@wildo-ai/saas-backend-lib 1.1.2 → 1.1.3

package/dist/esm/queue/job-consumer.backend.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"job-consumer.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/job-consumer.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAEzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AACnL,OAAO,EAAE,kCAAkC,EAAE,gCAAgC,EAAE,MAAM,+DAA+D,CAAC;AACrJ,OAAO,EAAE,6BAA6B,EAAkB,MAAM,6CAA6C,CAAC;AAC5G,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAC;AACtG,OAAO,EAGL,SAAS,EAET,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EAEtB,SAAS,EACT,2BAA2B,EAC3B,aAAa,GAEd,MAAM,uBAAuB,CAAC;AAqB/B,MAAM,uBAAuB,GAAsB;IACjD,WAAW,EAAE,IAAI;IACjB,WAAW,EAAE,oBAAoB;CAClC,CAAC;AAGK,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IASjB;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAxBT,QAAQ,CAA+D;IACzE,cAAc,GAAkB,IAAI,CAAC;IAC5B,cAAc,CAAiB;IACxC,WAAW,GAAG,KAAK,CAAC;IACpB,eAAe,GAAa,EAAE,CAAC;IAEvC,YAEmB,gBAAyC,EAEzC,OAAsC,EAEtC,WAAsC,EAEtC,iBAAkD,EAElD,kBAAuD,EAEvD,eAAoD,EAEpD,wBAAgE,EAEhE,MAAqB,EAErB,YAAwC;QAhBxC,qBAAgB,GAAhB,gBAAgB,CAAyB;QAEzC,YAAO,GAAP,OAAO,CAA+B;QAEtC,gBAAW,GAAX,WAAW,CAA2B;QAEtC,sBAAiB,GAAjB,iBAAiB,CAAiC;QAElD,uBAAkB,GAAlB,kBAAkB,CAAqC;QAEvD,oBAAe,GAAf,eAAe,CAAqC;QAEpD,6BAAwB,GAAxB,wBAAwB,CAAwC;QAEhE,WAAM,GAAN,MAAM,CAAe;QAErB,iBAAY,GAAZ,YAAY,CAA4B;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QACpE,0EAA0E;QAC1E,IAAI,CAAC,cAAc,GAAG;YACpB,QAAQ,EAAE,MAAM,EAAE;YAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;YACvB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,IAAY,aAAa;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,SAA4B,EAAE;QACjD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,MAAM,YAAY,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;QAE/D,sBAAsB;QACtB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAE7B,uEAAuE;QACvE,MAAM,sBAAsB,GAAG;YAC7B,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE;YAC9C,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE;SAC3D,CAAC;QAEF,KAAK,MAAM,WAAW,IAAI,sBAAsB,EAAE,CAAC;YACjD,MAAM,SAAS,GAAG,gBAAgB,CAAC,cAAc,CAC/C,IAAI,CAAC,aAAa,EAClB,WAAW,CAAC,QAAQ,EACpB,WAAW,CAAC,OAAO,CACpB,CAAC;YAEF,uDAAuD;YACvD,MAAM,OAAO,GAAmB,KAAK,EAAE,GAAmB,EAAE,OAAgB,EAAE,EAAE;gBAC9E,8FAA8F;gBAC9F,kFAAkF;gBAClF,MAAM,gCAAgC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;YAC/F,CAAC,CAAC;YAEF,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC/C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAErC,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;gBAC5C,SAAS;gBACT,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;aACvC,CAAC,CAAC;QACL,CAAC;QAED,mFAAmF;QACnF,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QAE5F,MAAM,gBAAgB,GAAmB,KAAK,EAAE,GAAmB,EAAE,OAAgB,EAAE,EAAE;YACvF,wFAAwF;YACxF,MAAM,gCAAgC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACxG,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE9C,IAAI,CAAC,QAAQ,CAAC,wCAAwC,EAAE;YACtD,SAAS,EAAE,kBAAkB;YAC7B,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;YACtC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;SACpC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7C,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;SACvC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,GAAmB,EACnB,OAAgB,EAChB,MAAyB;QAEzB,IAAI,GAAe,CAAC;QAEpB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE;gBAC/C,GAAG,2BAA2B,CAAC,UAAU,CAAC;aAC3C,CAAC,CAAC;YACH,kCAAkC;YAClC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAED,gGAAgG;QAChG,kGAAkG;QAClG,iGAAiG;QACjG,kGAAkG;QAClG,kCAAkC,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAE5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,4CAA4C;QAC5C,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,sBAAsB;YACtB,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAElF,4EAA4E;YAC5E,mEAAmE;YACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEnD,kFAAkF;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;gBACvC,oDAAoD;gBACpD,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC1E,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEtD,8BAA8B;YAC9B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAED,iEAAiE;YACjE,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC;gBACrC,MAAM;gBACN,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;gBACtC,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC;YAEH,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gFAAgF;YAChF,6EAA6E;YAC7E,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACzE,CAAC;gBAAS,CAAC;YACT,oDAAoD;YACpD,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB,CAC5B,GAAe,EACf,MAAiB,EACjB,GAAmB,EACnB,OAAgB,EAChB,MAAyB,EACzB,SAAiB;QAEjB,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU,CAAC;QACvG,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,iDAAiD;QACjD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW;YACpC,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QAExE,IAAI,WAAW,EAAE,CAAC;YAChB,4EAA4E;YAC5E,2EAA2E;YAC3E,sEAAsE;YACtE,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC;YAChC,MAAM,WAAW,GAAqB;gBACpC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,UAAU;gBACtB,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBACpG,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,IAAI,oBAAoB,CAAC,SAAS;aAC5F,CAAC;YAEF,MAAM,KAAK,GAAG,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAEtE,kDAAkD;YAClD,MAAM,QAAQ,GAAe;gBAC3B,GAAG,GAAG;gBACN,UAAU,EAAE,YAAY,GAAG,CAAC;aAC7B,CAAC;YAEF,4EAA4E;YAC5E,0EAA0E;YAC1E,+CAA+C;YAC/C,MAAM,eAAe,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC1F,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE;oBACxE,OAAO;oBACP,aAAa;iBACd,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,wEAAwE;gBACxE,uDAAuD;gBACvD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;oBACzE,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,GAAG,2BAA2B,CAAC,aAAa,CAAC;iBAC9C,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,8EAA8E;YAC9E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,gFAAgF;YAChF,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,CAAC;gBAAC,OAAO,aAAa,EAAE,CAAC;oBACvB,IAAI,CAAC,QAAQ,CAAC,yCAAyC,EAAE;wBACvD,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,KAAK,EAAE,mBAAmB,CAAC,aAAa,CAAC;qBAC1C,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBAC5D,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY,GAAG,CAAC;gBAC5B,UAAU;gBACV,OAAO,EAAE,KAAK;gBACd,yEAAyE;gBACzE,0EAA0E;gBAC1E,wEAAwE;gBACxE,4CAA4C;gBAC5C,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QAEL,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEzD,0BAA0B;YAC1B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAED,qEAAqE;YACrE,2EAA2E;YAC3E,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY;gBACxB,UAAU;gBACV,gEAAgE;gBAChE,uEAAuE;gBACvE,uEAAuE;gBACvE,kCAAkC;gBAClC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,QAA4C,EAC5C,YAAoB,EACpB,UAAkB;QAElB,0BAA0B;QAC1B,IAAI,YAAY,IAAI,UAAU,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,6EAA6E;QAC7E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,sBAAsB,CAAC,KAAK,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAC1B,GAAe,EACf,KAAc,EACd,GAAmB,EACnB,OAAgB,EAChB,MAAyB,EACzB,SAAiB;QAEjB,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU,CAAC;QACvG,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,0EAA0E;QAC1E,4EAA4E;QAC5E,4EAA4E;QAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,cAAc,GAA4B;YAC9C,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,GAAG,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,SAAS,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC;QAEF,2BAA2B;QAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QAE5F,IAAI,WAAW,EAAE,CAAC;YAChB,sEAAsE;YACtE,wDAAwD;YACxD,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC;YAChC,MAAM,WAAW,GAAqB;gBACpC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,UAAU;gBACtB,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBACpG,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,IAAI,oBAAoB,CAAC,SAAS;aAC5F,CAAC;YAEF,MAAM,KAAK,GAAG,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAEtE,kDAAkD;YAClD,MAAM,QAAQ,GAAe;gBAC3B,GAAG,GAAG;gBACN,UAAU,EAAE,YAAY,GAAG,CAAC;aAC7B,CAAC;YAEF,qEAAqE;YACrE,MAAM,eAAe,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC1F,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE;oBACxE,OAAO;oBACP,aAAa;iBACd,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;oBACzE,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,GAAG,2BAA2B,CAAC,aAAa,CAAC;iBAC9C,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,8EAA8E;YAC9E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBAC5D,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY,GAAG,CAAC;gBAC5B,UAAU;gBACV,OAAO,EAAE,KAAK;gBACd,GAAG,cAAc;aAClB,CAAC,CAAC;QAEL,CAAC;aAAM,CAAC;YACN,8CAA8C;YAE9C,wBAAwB;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEzD,0BAA0B;YAC1B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAc;oBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,KAAK,EAAE;wBACL,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,QAAQ;wBAC1C,QAAQ,EAAE,aAAa,CAAC,IAAI;wBAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;wBACrB,OAAO,EAAE,SAAS,CAAC,OAAO;wBAC1B,cAAc,EAAE,EAAE;wBAClB,sBAAsB,EAAE,2BAA2B,CAAC,QAAQ;wBAC5D,UAAU,EAAE,GAAG;qBAChB;oBACD,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;oBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACnC,CAAC;gBAEF,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YAED,qEAAqE;YACrE,2EAA2E;YAC3E,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY;gBACxB,UAAU;gBACV,GAAG,cAAc;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,YAAoB,EAAE,UAAkB;QAC1E,0BAA0B;QAC1B,IAAI,YAAY,IAAI,UAAU,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,qBAAqB,CAAC;YAC7C,6EAA6E;YAC7E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,sBAAsB,CAAC,KAAK,CAAC;QAC7E,CAAC;QAED,8BAA8B;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,KAAc;QAOjC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,MAAM,IAAI,KAAK;YACf,OAAQ,KAAa,CAAC,IAAI,KAAK,QAAQ,CACxC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAc;QAMrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;gBACpD,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,QAAQ;YACxB,OAAO,EAAE;gBACP,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC;aACpC;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAClC,GAAmB,EACnB,OAAgB,EAChB,MAAyB;QAEzB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAErC,4CAA4C;QAC5C,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,CAAC;YACH,iDAAiD;YACjD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ;gBAC3B,mBAAmB,EAAE,CAAC,CAAC,GAAG,CAAC,gBAAgB;gBAC3C,aAAa,EAAE,GAAG,CAAC,aAAa;aACjC,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,sBAAsB;YACtB,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAElF,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEnD,kFAAkF;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;gBACvC,6BAA6B;gBAC7B,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAM,CAAC,CAAC;gBAElE,oEAAoE;gBACpE,wEAAwE;gBACxE,gBAAgB;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;oBAClD,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC9D,CAAC,CAAC;gBAEH,yEAAyE;gBACzE,6DAA6D;gBAC7D,+DAA+D;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEtD,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;gBACvD,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,qEAAqE;YACrE,6EAA6E;YAC7E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;gBAClD,GAAG,2BAA2B,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;YAEH,yEAAyE;YACzE,6DAA6D;YAC7D,+DAA+D;YAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAElC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AAxnBY,yBAAyB;IADrC,UAAU,EAAE;IASR,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,yBAAyB,CAAC,CAAA;IAEpD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,4BAA4B,CAAC,CAAA;IAEvD,WAAA,MAAM,CAAC,kBAAkB,CAAC,4BAA4B,CAAC,CAAA;IAEvD,WAAA,MAAM,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAA;IAE1D,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAA;qCAfZ,uBAAuB;QAEhC,6BAA6B;QAEzB,yBAAyB;QAEnB,+BAA+B;QAE9B,mCAAmC;QAEtC,mCAAmC;QAE1B,sCAAsC;QAExD,aAAa;QAEP,0BAA0B;GAzBhD,yBAAyB,CAwnBrC","sourcesContent":["/**\n * Job Consumer Backend Service\n *\n * Handles the complete message lifecycle:\n * - Consumes messages from RabbitMQ\n * - Executes jobs via JobExecutorBackendService\n * - Handles retry logic with exponential backoff\n * - Sends RPC replies for QUEUED mode\n * - Tracks job status\n * - Acknowledges/rejects messages appropriately\n */\n\nimport { injectable, inject } from 'inversify';\nimport { v4 as uuidv4 } from 'uuid';\nimport * as os from 'os';\nimport type { ConsumeMessage, Channel } from 'amqplib';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';\nimport { projectWildoBackendErrorLogFields, describeCaughtError, projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport { adoptCorrelationIdIntoCurrentFrame, runInFreshRequestContextIfAbsent } from '~wildo-backend-lib/runtime/models/request-context.als.backend';\nimport { RabbitMQAdapterBackendService, MessageHandler } from './adapters/rabbitmq-adapter.backend.service';\nimport { JobExecutorBackendService } from './job-executor.backend.service';\nimport { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport { JobCompletionRegistryBackendService } from './job-completion-registry.backend.service';\nimport { WorkerShutdownManagerBackendService } from './worker-shutdown-manager.backend.service';\nimport { ScheduledMessageVerifierBackendService } from './scheduled-message-verifier.backend.service';\nimport {\n  JobPayload,\n  JobResult,\n  JobStatus,\n  JobPriority,\n  JobRetryUtils,\n  QueueNamingUtils,\n  QueueVariant,\n  QUEUE_RETRY_DEFAULTS,\n  ErrorHandling_Strategy,\n  QueueRetryConfig,\n  ErrorType,\n  ErrorCustomMessageReference,\n  ErrorSeverity,\n  ErrorActionBackend,\n} from '@wildo-ai/saas-models';\n\n/**\n * Worker identification for tracking.\n */\ninterface WorkerIdentity {\n  workerId: string;\n  hostname: string;\n  pid: number;\n}\n\n/**\n * Consumer configuration.\n */\nexport interface JobConsumerConfig {\n  /** Whether to enable retry logic */\n  enableRetry?: boolean;\n  /** Default retry configuration */\n  retryConfig?: Partial<QueueRetryConfig>;\n}\n\nconst DEFAULT_CONSUMER_CONFIG: JobConsumerConfig = {\n  enableRetry: true,\n  retryConfig: QUEUE_RETRY_DEFAULTS,\n};\n\n@injectable()\nexport class JobConsumerBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private _applicationId: string | null = null;\n  private readonly workerIdentity: WorkerIdentity;\n  private isConsuming = false;\n  private consumingQueues: string[] = [];\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.AppConfigService)\n    private readonly appConfigService: AppConfigurationService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly adapter: RabbitMQAdapterBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobExecutorBackendService)\n    private readonly jobExecutor: JobExecutorBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobStatusTrackingService)\n    private readonly jobStatusTracking: JobStatusTrackingBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobCompletionRegistryService)\n    private readonly completionRegistry: JobCompletionRegistryBackendService,\n    @inject(SAAS_SERVICE_TYPES.WorkerShutdownManagerService)\n    private readonly shutdownManager: WorkerShutdownManagerBackendService,\n    @inject(SAAS_SERVICE_TYPES.ScheduledMessageVerifierService)\n    private readonly scheduledMessageVerifier: ScheduledMessageVerifierBackendService,\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)\n    private readonly errorBuilder: ErrorBuilderBackendService,\n  ) {\n    this.logDebug = this.logger.debugByType('JobConsumerBackendService');\n    this.logger.setDebugTypeLogging('JobConsumerBackendService', false);\n    // Lazy-load applicationId to avoid accessing config before initialization\n    this.workerIdentity = {\n      workerId: uuidv4(),\n      hostname: os.hostname(),\n      pid: process.pid,\n    };\n  }\n\n  /** Lazy getter for applicationId - avoids accessing config in constructor */\n  private get applicationId(): string {\n    if (!this._applicationId) {\n      this._applicationId = this.appConfigService.applicationId;\n    }\n    return this._applicationId;\n  }\n\n  /**\n   * Start consuming from all operation and scheduled queues.\n   */\n  async startConsuming(config: JobConsumerConfig = {}): Promise<void> {\n    if (this.isConsuming) {\n      this.logger.warn('Consumer already running');\n      return;\n    }\n\n    const mergedConfig = { ...DEFAULT_CONSUMER_CONFIG, ...config };\n\n    // Connect to RabbitMQ\n    await this.adapter.connect();\n\n    // Start consuming from operations queues (normal and priority variant)\n    const operationsQueueConfigs = [\n      { category: 'operations', variant: undefined },\n      { category: 'operations', variant: QueueVariant.PRIORITY },\n    ];\n\n    for (const queueConfig of operationsQueueConfigs) {\n      const queueName = QueueNamingUtils.buildQueueName(\n        this.applicationId,\n        queueConfig.category,\n        queueConfig.variant\n      );\n\n      // Create handler that matches MessageHandler signature\n      const handler: MessageHandler = async (msg: ConsumeMessage, channel: Channel) => {\n        // Phase 1 — each consumed job runs in its own correlation frame (fresh id). Phase 6 will seed\n        // it from the enqueuing request's originCorrelationId carried on the job payload.\n        await runInFreshRequestContextIfAbsent(() => this.handleMessage(msg, channel, mergedConfig));\n      };\n\n      await this.adapter.consume(queueName, handler);\n      this.consumingQueues.push(queueName);\n\n      this.logDebug('Started consuming from queue', {\n        queueName,\n        workerId: this.workerIdentity.workerId,\n      });\n    }\n\n    // Start consuming from scheduled queue (JWT-signed messages from crontabs-manager)\n    const scheduledQueueName = QueueNamingUtils.buildQueueName(this.applicationId, 'scheduled');\n\n    const scheduledHandler: MessageHandler = async (msg: ConsumeMessage, channel: Channel) => {\n      // Phase 1 — each cron tick (scheduled-queue message) runs in its own correlation frame.\n      await runInFreshRequestContextIfAbsent(() => this.handleScheduledMessage(msg, channel, mergedConfig));\n    };\n\n    await this.adapter.consume(scheduledQueueName, scheduledHandler);\n    this.consumingQueues.push(scheduledQueueName);\n\n    this.logDebug('Started consuming from scheduled queue', {\n      queueName: scheduledQueueName,\n      workerId: this.workerIdentity.workerId,\n    });\n\n    this.isConsuming = true;\n    this.logDebug('Job consumer started', {\n      workerId: this.workerIdentity.workerId,\n      queues: this.consumingQueues.length,\n    });\n  }\n\n  /**\n   * Stop consuming and close connections.\n   */\n  async stopConsuming(): Promise<void> {\n    if (!this.isConsuming) {\n      return;\n    }\n\n    // Cancel all consumers\n    for (const queueName of this.consumingQueues) {\n      await this.adapter.cancelConsumer(queueName);\n    }\n\n    this.consumingQueues = [];\n    this.isConsuming = false;\n\n    this.logDebug('Job consumer stopped', {\n      workerId: this.workerIdentity.workerId,\n    });\n  }\n\n  /**\n   * Handle a single message from the queue.\n   */\n  private async handleMessage(\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig\n  ): Promise<void> {\n    let job: JobPayload;\n\n    try {\n      job = JSON.parse(msg.content.toString());\n    } catch (parseError) {\n      this.logger.error('Failed to parse job message', {\n        ...projectCaughtErrorLogFields(parseError),\n      });\n      // Invalid message - don't requeue\n      channel.nack(msg, false, false);\n      return;\n    }\n\n    // Phase 6 — this job belongs to the trace of the request that enqueued it: adopt that request's\n    // originCorrelationId into the (freshly-minted) consume frame, so every log line + audit row this\n    // job produces on the worker carries the originating request's correlationId. Absent/malformed →\n    // the frame keeps its fresh id (a job with no originating request stays independently traceable).\n    adoptCorrelationIdIntoCurrentFrame(job.originCorrelationId);\n\n    const startTime = Date.now();\n    const { replyTo, correlationId } = msg.properties;\n\n    // Track in-flight job for graceful shutdown\n    this.shutdownManager.jobStarted();\n\n    this.logDebug('Processing job', {\n      jobId: job.jobId,\n      jobType: job.jobType,\n      priority: job.priority,\n      retryCount: job.retryCount ?? 0,\n    });\n\n    try {\n      // Mark job as started\n      await this.jobStatusTracking.markStarted(job.jobId, this.workerIdentity.workerId);\n\n      // Execute the job - returns JobResult which may indicate success OR failure\n      // The executor catches all errors and returns a result with status\n      const result = await this.jobExecutor.execute(job);\n\n      // Check if job failed (executor returns result with FAILED status, doesn't throw)\n      if (result.status === JobStatus.FAILED) {\n        // Convert the error result to proper retry handling\n        await this.handleJobFailure(job, result, msg, channel, config, startTime);\n        return;\n      }\n\n      // Mark as completed only on success\n      await this.jobStatusTracking.markCompleted(job.jobId);\n\n      // Send RPC reply if requested\n      if (replyTo && correlationId) {\n        await this.adapter.publishResult(replyTo, correlationId, result);\n      }\n\n      // Dispatch to completion handlers (for BACKGROUND_JOB callbacks)\n      await this.completionRegistry.dispatch({\n        result,\n        operationPathKey: job.operationPathKey,\n        batchRef: job.batchRef,\n      });\n\n      // Acknowledge success\n      channel.ack(msg);\n\n      this.logDebug('Job completed successfully', {\n        jobId: job.jobId,\n        durationMs: Date.now() - startTime,\n      });\n\n    } catch (error) {\n      // This catch block handles unexpected errors (e.g., from markStarted, dispatch)\n      // NOT job execution errors (those are returned by executor as FAILED status)\n      await this.handleJobError(job, error, msg, channel, config, startTime);\n    } finally {\n      // Always track job completion for graceful shutdown\n      this.shutdownManager.jobCompleted();\n    }\n  }\n\n  /**\n   * Handle job failure when executor returns FAILED status (not an exception).\n   * The executor catches errors and returns a JobResult with status FAILED.\n   */\n  private async handleJobFailure(\n    job: JobPayload,\n    result: JobResult,\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig,\n    startTime: number\n  ): Promise<void> {\n    const currentRetry = job.retryCount ?? 0;\n    const maxRetries = job.maxRetries ?? config.retryConfig?.maxRetries ?? QUEUE_RETRY_DEFAULTS.maxRetries;\n    const { replyTo, correlationId } = msg.properties;\n\n    // Extract error handling strategy from the error\n    const errorStrategy = result.error?.errorHandlingStrategy;\n    const shouldRetry = config.enableRetry &&\n      this.shouldRetryFromStrategy(errorStrategy, currentRetry, maxRetries);\n\n    if (shouldRetry) {\n      // Resolve backoff from the job's per-operation retryConfig, falling back to\n      // the consumer config then the framework defaults (per-field, so a partial\n      // authored retryConfig still gets sane values for the axes it omits).\n      const backoff = job.retryConfig;\n      const retryConfig: QueueRetryConfig = {\n        retryPolicy: backoff?.retryPolicy ?? config.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n        maxRetries: maxRetries,\n        baseDelayMs: backoff?.baseDelayMs ?? config.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n        maxDelayMs: backoff?.maxDelayMs ?? config.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n        timeoutMs: job.timeoutMs ?? config.retryConfig?.timeoutMs ?? QUEUE_RETRY_DEFAULTS.timeoutMs,\n      };\n\n      const delay = JobRetryUtils.calculateDelay(retryConfig, currentRetry);\n\n      // Create updated job with incremented retry count\n      const retryJob: JobPayload = {\n        ...job,\n        retryCount: currentRetry + 1,\n      };\n\n      // Durable backoff: hand the retry to the broker (delay queue holds it, then\n      // dead-letters it back) instead of an in-process setTimeout that would be\n      // lost if this worker crashes during the wait.\n      const operationsQueue = QueueNamingUtils.buildQueueName(this.applicationId, 'operations');\n      try {\n        await this.adapter.scheduleDelayedRetry(operationsQueue, retryJob, delay, {\n          replyTo,\n          correlationId,\n        });\n      } catch (scheduleError) {\n        // Could not hand the retry to the broker — requeue the ORIGINAL (do not\n        // ACK) so it is redelivered rather than silently lost.\n        this.logger.error('Failed to schedule durable retry; requeueing original', {\n          jobId: job.jobId,\n          ...projectCaughtErrorLogFields(scheduleError),\n        });\n        channel.nack(msg, false, true);\n        return;\n      }\n\n      // ACK original message (retry is now durably owned by the broker delay queue)\n      channel.ack(msg);\n\n      // Best-effort tracking update → RETRYING + lastRetryAt (never block the retry).\n      if (result.error) {\n        try {\n          await this.jobStatusTracking.markFailed(job.jobId, result.error);\n        } catch (trackingError) {\n          this.logDebug('Failed to mark job retrying (non-fatal)', {\n            jobId: job.jobId,\n            error: describeCaughtError(trackingError),\n          });\n        }\n      }\n\n      this.logger.warn('Job retry scheduled (durable delay queue)', {\n        jobId: job.jobId,\n        retryCount: currentRetry + 1,\n        maxRetries,\n        delayMs: delay,\n        // The queue is a non-HTTP boundary and owns its own operational level (a\n        // scheduled retry is a `warn`, not classified from the error's severity).\n        // Reuse the canonical field projection to restore the full error detail\n        // that no longer self-logs at construction.\n        ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n      });\n\n    } else {\n      // Max retries exceeded or non-retryable error\n      await this.jobStatusTracking.markDeadLettered(job.jobId);\n\n      // Send error reply if RPC\n      if (replyTo && correlationId) {\n        await this.adapter.publishResult(replyTo, correlationId, result);\n      }\n\n      // Reject without requeue → the operations queue's DLX now parks this\n      // permanently-failed job in `operations.deadletter` (previously the broker\n      // discarded it because no dead-letter-exchange was wired).\n      channel.nack(msg, false, false);\n\n      this.logger.error('Job failed permanently', {\n        jobId: job.jobId,\n        retryCount: currentRetry,\n        maxRetries,\n        // Permanent (dead-lettered) failure is operationally an `error`\n        // regardless of the underlying error's business severity, so the level\n        // stays explicit; the projection restores the full error detail (type,\n        // status, context) and the stack.\n        ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n        ...(result.error?.stack ? { stack: result.error.stack } : {}),\n      });\n    }\n  }\n\n  /**\n   * Check if retry should happen based on error strategy.\n   */\n  private shouldRetryFromStrategy(\n    strategy: ErrorHandling_Strategy | undefined,\n    currentRetry: number,\n    maxRetries: number\n  ): boolean {\n    // Check retry count limit\n    if (currentRetry >= maxRetries) {\n      return false;\n    }\n\n    // Only retry if strategy is explicitly RETRY or undefined (default to retry)\n    return strategy === undefined || strategy === ErrorHandling_Strategy.RETRY;\n  }\n\n  /**\n   * Handle unexpected errors (not job execution failures).\n   * This handles errors from status tracking, dispatch, etc.\n   */\n  private async handleJobError(\n    job: JobPayload,\n    error: unknown,\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig,\n    startTime: number\n  ): Promise<void> {\n    const currentRetry = job.retryCount ?? 0;\n    const maxRetries = job.maxRetries ?? config.retryConfig?.maxRetries ?? QUEUE_RETRY_DEFAULTS.maxRetries;\n    const { replyTo, correlationId } = msg.properties;\n\n    // This is a raw caught exception (not a returned JobResult). Normalize it\n    // once so both the retry and permanent-failure logs carry the full detail —\n    // errors no longer self-log at construction, so the queue boundary owns it.\n    const errorInfo = this.extractErrorInfo(error);\n    const errorLogFields: Record<string, unknown> = {\n      errorType: errorInfo.type,\n      ...(errorInfo.customMessageReference ? { customMessageReference: errorInfo.customMessageReference } : {}),\n      ...(errorInfo.statusCode ? { statusCode: errorInfo.statusCode } : {}),\n      ...(errorInfo.context ? { context: errorInfo.context } : {}),\n    };\n\n    // Check if we should retry\n    const shouldRetry = config.enableRetry && this.shouldRetry(error, currentRetry, maxRetries);\n\n    if (shouldRetry) {\n      // Resolve backoff from the job's per-operation retryConfig (per-field\n      // fallback to consumer config then framework defaults).\n      const backoff = job.retryConfig;\n      const retryConfig: QueueRetryConfig = {\n        retryPolicy: backoff?.retryPolicy ?? config.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n        maxRetries: maxRetries,\n        baseDelayMs: backoff?.baseDelayMs ?? config.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n        maxDelayMs: backoff?.maxDelayMs ?? config.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n        timeoutMs: job.timeoutMs ?? config.retryConfig?.timeoutMs ?? QUEUE_RETRY_DEFAULTS.timeoutMs,\n      };\n\n      const delay = JobRetryUtils.calculateDelay(retryConfig, currentRetry);\n\n      // Create updated job with incremented retry count\n      const retryJob: JobPayload = {\n        ...job,\n        retryCount: currentRetry + 1,\n      };\n\n      // Durable backoff via the broker delay queue (see handleJobFailure).\n      const operationsQueue = QueueNamingUtils.buildQueueName(this.applicationId, 'operations');\n      try {\n        await this.adapter.scheduleDelayedRetry(operationsQueue, retryJob, delay, {\n          replyTo,\n          correlationId,\n        });\n      } catch (scheduleError) {\n        this.logger.error('Failed to schedule durable retry; requeueing original', {\n          jobId: job.jobId,\n          ...projectCaughtErrorLogFields(scheduleError),\n        });\n        channel.nack(msg, false, true);\n        return;\n      }\n\n      // ACK original message (retry is now durably owned by the broker delay queue)\n      channel.ack(msg);\n\n      this.logger.warn('Job retry scheduled (durable delay queue)', {\n        jobId: job.jobId,\n        retryCount: currentRetry + 1,\n        maxRetries,\n        delayMs: delay,\n        ...errorLogFields,\n      });\n\n    } else {\n      // Max retries exceeded or non-retryable error\n\n      // Mark as dead-lettered\n      await this.jobStatusTracking.markDeadLettered(job.jobId);\n\n      // Send error reply if RPC\n      if (replyTo && correlationId) {\n        const jobResult: JobResult = {\n          jobId: job.jobId,\n          status: JobStatus.FAILED,\n          error: {\n            type: errorInfo.type ?? ErrorType.INTERNAL,\n            severity: ErrorSeverity.HIGH,\n            timestamp: new Date(),\n            context: errorInfo.context,\n            backendActions: [],\n            customMessageReference: ErrorCustomMessageReference.INTERNAL,\n            statusCode: 500,\n          },\n          startedAt: new Date(startTime),\n          completedAt: new Date(),\n          durationMs: Date.now() - startTime,\n        };\n\n        await this.adapter.publishResult(replyTo, correlationId, jobResult);\n      }\n\n      // Reject without requeue → the operations queue's DLX now parks this\n      // permanently-failed job in `operations.deadletter` (previously the broker\n      // discarded it because no dead-letter-exchange was wired).\n      channel.nack(msg, false, false);\n\n      this.logger.error('Job failed permanently', {\n        jobId: job.jobId,\n        retryCount: currentRetry,\n        maxRetries,\n        ...errorLogFields,\n      });\n    }\n  }\n\n  /**\n   * Determine if an error is retryable.\n   */\n  private shouldRetry(error: unknown, currentRetry: number, maxRetries: number): boolean {\n    // Check retry count limit\n    if (currentRetry >= maxRetries) {\n      return false;\n    }\n\n    // Check if error specifies non-retryable\n    if (this.isWildoError(error)) {\n      const strategy = error.errorHandlingStrategy;\n      // Only retry if strategy is explicitly RETRY or undefined (default to retry)\n      return strategy === undefined || strategy === ErrorHandling_Strategy.RETRY;\n    }\n\n    // Default: retry on any error\n    return true;\n  }\n\n  /**\n   * Type guard for WildoError_Backend.\n   */\n  private isWildoError(error: unknown): error is {\n    type: ErrorType;\n    errorHandlingStrategy?: ErrorHandling_Strategy;\n    customMessageReference?: ErrorCustomMessageReference;\n    statusCode?: number;\n    context?: Record<string, unknown>;\n  } {\n    return (\n      typeof error === 'object' &&\n      error !== null &&\n      'type' in error &&\n      typeof (error as any).type === 'string'\n    );\n  }\n\n  /**\n   * Extract error information for result/logging.\n   */\n  private extractErrorInfo(error: unknown): {\n    type?: ErrorType;\n    customMessageReference?: ErrorCustomMessageReference;\n    statusCode?: number;\n    context?: Record<string, unknown>;\n  } {\n    if (this.isWildoError(error)) {\n      return {\n        type: error.type,\n        customMessageReference: error.customMessageReference,\n        statusCode: error.statusCode,\n        context: error.context,\n      };\n    }\n\n    return {\n      type: ErrorType.INTERNAL,\n      context: {\n        message: describeCaughtError(error),\n      },\n    };\n  }\n\n  /**\n   * Handle a scheduled message (JWT-signed from crontabs-batches-manager).\n   */\n  private async handleScheduledMessage(\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig\n  ): Promise<void> {\n    const token = msg.content.toString();\n\n    // Track in-flight job for graceful shutdown\n    this.shutdownManager.jobStarted();\n\n    try {\n      // Verify the JWT token and convert to JobPayload\n      const job = await this.scheduledMessageVerifier.verifyAndConvert(token);\n\n      this.logger.info('Processing scheduled job', {\n        jobId: job.jobId,\n        hasBatchRef: !!job.batchRef,\n        hasOperationPathKey: !!job.operationPathKey,\n        applicationId: job.applicationId,\n      });\n\n      const startTime = Date.now();\n\n      // Mark job as started\n      await this.jobStatusTracking.markStarted(job.jobId, this.workerIdentity.workerId);\n\n      // Execute the job - returns JobResult which may indicate success OR failure\n      const result = await this.jobExecutor.execute(job);\n\n      // Check if job failed (executor returns result with FAILED status, doesn't throw)\n      if (result.status === JobStatus.FAILED) {\n        // Mark as failed in tracking\n        await this.jobStatusTracking.markFailed(job.jobId, result.error!);\n\n        // Log failure — the queue owns the operational level (`error`); the\n        // projection restores the full error detail that no longer self-logs at\n        // construction.\n        this.logger.error('Scheduled job execution failed', {\n          jobId: job.jobId,\n          durationMs: Date.now() - startTime,\n          ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n          ...(result.error?.stack ? { stack: result.error.stack } : {}),\n        });\n\n        // Reject without requeue: scheduled jobs don't retry via the queue (they\n        // re-run on the next schedule). The message is now parked in\n        // `scheduled.deadletter` as a stale audit copy for inspection.\n        channel.nack(msg, false, false);\n        return;\n      }\n\n      // Mark as completed only on success\n      await this.jobStatusTracking.markCompleted(job.jobId);\n\n      // Acknowledge success\n      channel.ack(msg);\n\n      this.logger.info('Scheduled job completed successfully', {\n        jobId: job.jobId,\n        durationMs: Date.now() - startTime,\n      });\n\n    } catch (error) {\n      // This catches errors from token verification, status tracking, etc.\n      // NOT job execution errors (those are returned by executor as FAILED status)\n      this.logger.error('Scheduled job processing error', {\n        ...projectCaughtErrorLogFields(error),\n      });\n\n      // Reject without requeue: scheduled jobs don't retry via the queue (they\n      // re-run on the next schedule). The message is now parked in\n      // `scheduled.deadletter` as a stale audit copy for inspection.\n      channel.nack(msg, false, false);\n\n    } finally {\n      this.shutdownManager.jobCompleted();\n    }\n  }\n\n  /**\n   * Get worker identity for external access.\n   */\n  getWorkerIdentity(): WorkerIdentity {\n    return { ...this.workerIdentity };\n  }\n\n  /**\n   * Check if consumer is running.\n   */\n  isRunning(): boolean {\n    return this.isConsuming;\n  }\n}\n"]}
+{"version":3,"file":"job-consumer.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/job-consumer.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAEzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AACnL,OAAO,EAAE,kCAAkC,EAAE,gCAAgC,EAAE,MAAM,+DAA+D,CAAC;AAGrJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAC;AACtG,OAAO,EAGL,SAAS,EAET,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EAEtB,SAAS,EACT,2BAA2B,EAC3B,aAAa,GAEd,MAAM,uBAAuB,CAAC;AAqB/B,MAAM,uBAAuB,GAAsB;IACjD,WAAW,EAAE,IAAI;IACjB,WAAW,EAAE,oBAAoB;CAClC,CAAC;AAGK,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IASjB;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAxBT,QAAQ,CAA+D;IACzE,cAAc,GAAkB,IAAI,CAAC;IAC5B,cAAc,CAAiB;IACxC,WAAW,GAAG,KAAK,CAAC;IACpB,eAAe,GAAa,EAAE,CAAC;IAEvC,YAEmB,gBAAyC,EAEzC,OAAqB,EAErB,WAAsC,EAEtC,iBAAkD,EAElD,kBAAuD,EAEvD,eAAoD,EAEpD,wBAAgE,EAEhE,MAAqB,EAErB,YAAwC;QAhBxC,qBAAgB,GAAhB,gBAAgB,CAAyB;QAEzC,YAAO,GAAP,OAAO,CAAc;QAErB,gBAAW,GAAX,WAAW,CAA2B;QAEtC,sBAAiB,GAAjB,iBAAiB,CAAiC;QAElD,uBAAkB,GAAlB,kBAAkB,CAAqC;QAEvD,oBAAe,GAAf,eAAe,CAAqC;QAEpD,6BAAwB,GAAxB,wBAAwB,CAAwC;QAEhE,WAAM,GAAN,MAAM,CAAe;QAErB,iBAAY,GAAZ,YAAY,CAA4B;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QACpE,0EAA0E;QAC1E,IAAI,CAAC,cAAc,GAAG;YACpB,QAAQ,EAAE,MAAM,EAAE;YAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;YACvB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,IAAY,aAAa;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,SAA4B,EAAE;QACjD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,MAAM,YAAY,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAAC;QAE/D,sBAAsB;QACtB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAE7B,uEAAuE;QACvE,MAAM,sBAAsB,GAAG;YAC7B,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE;YAC9C,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,QAAQ,EAAE;SAC3D,CAAC;QAEF,KAAK,MAAM,WAAW,IAAI,sBAAsB,EAAE,CAAC;YACjD,MAAM,SAAS,GAAG,gBAAgB,CAAC,cAAc,CAC/C,IAAI,CAAC,aAAa,EAClB,WAAW,CAAC,QAAQ,EACpB,WAAW,CAAC,OAAO,CACpB,CAAC;YAEF,uDAAuD;YACvD,MAAM,OAAO,GAAmB,KAAK,EAAE,GAAmB,EAAE,OAAgB,EAAE,EAAE;gBAC9E,8FAA8F;gBAC9F,kFAAkF;gBAClF,MAAM,gCAAgC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;YAC/F,CAAC,CAAC;YAEF,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAC/C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAErC,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAAE;gBAC5C,SAAS;gBACT,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;aACvC,CAAC,CAAC;QACL,CAAC;QAED,mFAAmF;QACnF,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QAE5F,MAAM,gBAAgB,GAAmB,KAAK,EAAE,GAAmB,EAAE,OAAgB,EAAE,EAAE;YACvF,wFAAwF;YACxF,MAAM,gCAAgC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACxG,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE9C,IAAI,CAAC,QAAQ,CAAC,wCAAwC,EAAE;YACtD,SAAS,EAAE,kBAAkB;YAC7B,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;YACtC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;SACpC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7C,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ;SACvC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,GAAmB,EACnB,OAAgB,EAChB,MAAyB;QAEzB,IAAI,GAAe,CAAC;QAEpB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE;gBAC/C,GAAG,2BAA2B,CAAC,UAAU,CAAC;aAC3C,CAAC,CAAC;YACH,kCAAkC;YAClC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAED,gGAAgG;QAChG,kGAAkG;QAClG,iGAAiG;QACjG,kGAAkG;QAClG,kCAAkC,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAE5D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,4CAA4C;QAC5C,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,sBAAsB;YACtB,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAElF,4EAA4E;YAC5E,mEAAmE;YACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEnD,kFAAkF;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;gBACvC,oDAAoD;gBACpD,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC1E,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEtD,8BAA8B;YAC9B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAED,iEAAiE;YACjE,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC;gBACrC,MAAM;gBACN,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;gBACtC,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC;YAEH,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gFAAgF;YAChF,6EAA6E;YAC7E,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACzE,CAAC;gBAAS,CAAC;YACT,oDAAoD;YACpD,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB,CAC5B,GAAe,EACf,MAAiB,EACjB,GAAmB,EACnB,OAAgB,EAChB,MAAyB,EACzB,SAAiB;QAEjB,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU,CAAC;QACvG,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,iDAAiD;QACjD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW;YACpC,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QAExE,IAAI,WAAW,EAAE,CAAC;YAChB,4EAA4E;YAC5E,2EAA2E;YAC3E,sEAAsE;YACtE,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC;YAChC,MAAM,WAAW,GAAqB;gBACpC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,UAAU;gBACtB,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBACpG,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,IAAI,oBAAoB,CAAC,SAAS;aAC5F,CAAC;YAEF,MAAM,KAAK,GAAG,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAEtE,kDAAkD;YAClD,MAAM,QAAQ,GAAe;gBAC3B,GAAG,GAAG;gBACN,UAAU,EAAE,YAAY,GAAG,CAAC;aAC7B,CAAC;YAEF,4EAA4E;YAC5E,0EAA0E;YAC1E,+CAA+C;YAC/C,MAAM,eAAe,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC1F,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE;oBACxE,OAAO;oBACP,aAAa;iBACd,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,wEAAwE;gBACxE,uDAAuD;gBACvD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;oBACzE,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,GAAG,2BAA2B,CAAC,aAAa,CAAC;iBAC9C,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,8EAA8E;YAC9E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,gFAAgF;YAChF,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,CAAC;gBAAC,OAAO,aAAa,EAAE,CAAC;oBACvB,IAAI,CAAC,QAAQ,CAAC,yCAAyC,EAAE;wBACvD,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,KAAK,EAAE,mBAAmB,CAAC,aAAa,CAAC;qBAC1C,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBAC5D,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY,GAAG,CAAC;gBAC5B,UAAU;gBACV,OAAO,EAAE,KAAK;gBACd,yEAAyE;gBACzE,0EAA0E;gBAC1E,wEAAwE;gBACxE,4CAA4C;gBAC5C,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QAEL,CAAC;aAAM,CAAC;YACN,8CAA8C;YAC9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEzD,0BAA0B;YAC1B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAED,qEAAqE;YACrE,2EAA2E;YAC3E,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY;gBACxB,UAAU;gBACV,gEAAgE;gBAChE,uEAAuE;gBACvE,uEAAuE;gBACvE,kCAAkC;gBAClC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,QAA4C,EAC5C,YAAoB,EACpB,UAAkB;QAElB,0BAA0B;QAC1B,IAAI,YAAY,IAAI,UAAU,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,6EAA6E;QAC7E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,sBAAsB,CAAC,KAAK,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAC1B,GAAe,EACf,KAAc,EACd,GAAmB,EACnB,OAAgB,EAChB,MAAyB,EACzB,SAAiB;QAEjB,MAAM,YAAY,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU,CAAC;QACvG,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC;QAElD,0EAA0E;QAC1E,4EAA4E;QAC5E,4EAA4E;QAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,cAAc,GAA4B;YAC9C,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,GAAG,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,SAAS,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC;QAEF,2BAA2B;QAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QAE5F,IAAI,WAAW,EAAE,CAAC;YAChB,sEAAsE;YACtE,wDAAwD;YACxD,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC;YAChC,MAAM,WAAW,GAAqB;gBACpC,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,UAAU;gBACtB,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBACxG,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBACpG,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,IAAI,oBAAoB,CAAC,SAAS;aAC5F,CAAC;YAEF,MAAM,KAAK,GAAG,aAAa,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAEtE,kDAAkD;YAClD,MAAM,QAAQ,GAAe;gBAC3B,GAAG,GAAG;gBACN,UAAU,EAAE,YAAY,GAAG,CAAC;aAC7B,CAAC;YAEF,qEAAqE;YACrE,MAAM,eAAe,GAAG,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC1F,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE;oBACxE,OAAO;oBACP,aAAa;iBACd,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;oBACzE,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,GAAG,2BAA2B,CAAC,aAAa,CAAC;iBAC9C,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,8EAA8E;YAC9E,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBAC5D,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY,GAAG,CAAC;gBAC5B,UAAU;gBACV,OAAO,EAAE,KAAK;gBACd,GAAG,cAAc;aAClB,CAAC,CAAC;QAEL,CAAC;aAAM,CAAC;YACN,8CAA8C;YAE9C,wBAAwB;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEzD,0BAA0B;YAC1B,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAc;oBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,KAAK,EAAE;wBACL,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,QAAQ;wBAC1C,QAAQ,EAAE,aAAa,CAAC,IAAI;wBAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;wBACrB,OAAO,EAAE,SAAS,CAAC,OAAO;wBAC1B,cAAc,EAAE,EAAE;wBAClB,sBAAsB,EAAE,2BAA2B,CAAC,QAAQ;wBAC5D,UAAU,EAAE,GAAG;qBAChB;oBACD,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC;oBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACnC,CAAC;gBAEF,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;YACtE,CAAC;YAED,qEAAqE;YACrE,2EAA2E;YAC3E,2DAA2D;YAC3D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAEhC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,YAAY;gBACxB,UAAU;gBACV,GAAG,cAAc;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,KAAc,EAAE,YAAoB,EAAE,UAAkB;QAC1E,0BAA0B;QAC1B,IAAI,YAAY,IAAI,UAAU,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,qBAAqB,CAAC;YAC7C,6EAA6E;YAC7E,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,sBAAsB,CAAC,KAAK,CAAC;QAC7E,CAAC;QAED,8BAA8B;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,KAAc;QAOjC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,MAAM,IAAI,KAAK;YACf,OAAQ,KAAa,CAAC,IAAI,KAAK,QAAQ,CACxC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAc;QAMrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;gBACpD,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,QAAQ;YACxB,OAAO,EAAE;gBACP,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC;aACpC;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAClC,GAAmB,EACnB,OAAgB,EAChB,MAAyB;QAEzB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAErC,4CAA4C;QAC5C,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,CAAC;YACH,iDAAiD;YACjD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAExE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC3C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ;gBAC3B,mBAAmB,EAAE,CAAC,CAAC,GAAG,CAAC,gBAAgB;gBAC3C,aAAa,EAAE,GAAG,CAAC,aAAa;aACjC,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,sBAAsB;YACtB,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAElF,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEnD,kFAAkF;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;gBACvC,6BAA6B;gBAC7B,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAM,CAAC,CAAC;gBAElE,oEAAoE;gBACpE,wEAAwE;gBACxE,gBAAgB;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;oBAClD,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,iCAAiC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC9D,CAAC,CAAC;gBAEH,yEAAyE;gBACzE,6DAA6D;gBAC7D,+DAA+D;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,oCAAoC;YACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEtD,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE;gBACvD,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,qEAAqE;YACrE,6EAA6E;YAC7E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;gBAClD,GAAG,2BAA2B,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;YAEH,yEAAyE;YACzE,6DAA6D;YAC7D,+DAA+D;YAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAElC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,CAAC;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;CACF,CAAA;AAxnBY,yBAAyB;IADrC,UAAU,EAAE;IASR,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,yBAAyB,CAAC,CAAA;IAEpD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,4BAA4B,CAAC,CAAA;IAEvD,WAAA,MAAM,CAAC,kBAAkB,CAAC,4BAA4B,CAAC,CAAA;IAEvD,WAAA,MAAM,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAA;IAE1D,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAA;qCAfZ,uBAAuB,UAI5B,yBAAyB;QAEnB,+BAA+B;QAE9B,mCAAmC;QAEtC,mCAAmC;QAE1B,sCAAsC;QAExD,aAAa;QAEP,0BAA0B;GAzBhD,yBAAyB,CAwnBrC","sourcesContent":["/**\n * Job Consumer Backend Service\n *\n * Handles the complete message lifecycle:\n * - Consumes messages from RabbitMQ\n * - Executes jobs via JobExecutorBackendService\n * - Handles retry logic with exponential backoff\n * - Sends RPC replies for QUEUED mode\n * - Tracks job status\n * - Acknowledges/rejects messages appropriately\n */\n\nimport { injectable, inject } from 'inversify';\nimport { v4 as uuidv4 } from 'uuid';\nimport * as os from 'os';\nimport type { ConsumeMessage, Channel } from 'amqplib';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';\nimport { projectWildoBackendErrorLogFields, describeCaughtError, projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport { adoptCorrelationIdIntoCurrentFrame, runInFreshRequestContextIfAbsent } from '~wildo-backend-lib/runtime/models/request-context.als.backend';\nimport type { QueueAdapter } from './adapters/queue-adapter.backend.interface';\nimport type { MessageHandler } from './adapters/queue-adapter.backend.interface';\nimport { JobExecutorBackendService } from './job-executor.backend.service';\nimport { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport { JobCompletionRegistryBackendService } from './job-completion-registry.backend.service';\nimport { WorkerShutdownManagerBackendService } from './worker-shutdown-manager.backend.service';\nimport { ScheduledMessageVerifierBackendService } from './scheduled-message-verifier.backend.service';\nimport {\n  JobPayload,\n  JobResult,\n  JobStatus,\n  JobPriority,\n  JobRetryUtils,\n  QueueNamingUtils,\n  QueueVariant,\n  QUEUE_RETRY_DEFAULTS,\n  ErrorHandling_Strategy,\n  QueueRetryConfig,\n  ErrorType,\n  ErrorCustomMessageReference,\n  ErrorSeverity,\n  ErrorActionBackend,\n} from '@wildo-ai/saas-models';\n\n/**\n * Worker identification for tracking.\n */\ninterface WorkerIdentity {\n  workerId: string;\n  hostname: string;\n  pid: number;\n}\n\n/**\n * Consumer configuration.\n */\nexport interface JobConsumerConfig {\n  /** Whether to enable retry logic */\n  enableRetry?: boolean;\n  /** Default retry configuration */\n  retryConfig?: Partial<QueueRetryConfig>;\n}\n\nconst DEFAULT_CONSUMER_CONFIG: JobConsumerConfig = {\n  enableRetry: true,\n  retryConfig: QUEUE_RETRY_DEFAULTS,\n};\n\n@injectable()\nexport class JobConsumerBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private _applicationId: string | null = null;\n  private readonly workerIdentity: WorkerIdentity;\n  private isConsuming = false;\n  private consumingQueues: string[] = [];\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.AppConfigService)\n    private readonly appConfigService: AppConfigurationService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly adapter: QueueAdapter,\n    @inject(SAAS_SERVICE_TYPES.JobExecutorBackendService)\n    private readonly jobExecutor: JobExecutorBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobStatusTrackingService)\n    private readonly jobStatusTracking: JobStatusTrackingBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobCompletionRegistryService)\n    private readonly completionRegistry: JobCompletionRegistryBackendService,\n    @inject(SAAS_SERVICE_TYPES.WorkerShutdownManagerService)\n    private readonly shutdownManager: WorkerShutdownManagerBackendService,\n    @inject(SAAS_SERVICE_TYPES.ScheduledMessageVerifierService)\n    private readonly scheduledMessageVerifier: ScheduledMessageVerifierBackendService,\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)\n    private readonly errorBuilder: ErrorBuilderBackendService,\n  ) {\n    this.logDebug = this.logger.debugByType('JobConsumerBackendService');\n    this.logger.setDebugTypeLogging('JobConsumerBackendService', false);\n    // Lazy-load applicationId to avoid accessing config before initialization\n    this.workerIdentity = {\n      workerId: uuidv4(),\n      hostname: os.hostname(),\n      pid: process.pid,\n    };\n  }\n\n  /** Lazy getter for applicationId - avoids accessing config in constructor */\n  private get applicationId(): string {\n    if (!this._applicationId) {\n      this._applicationId = this.appConfigService.applicationId;\n    }\n    return this._applicationId;\n  }\n\n  /**\n   * Start consuming from all operation and scheduled queues.\n   */\n  async startConsuming(config: JobConsumerConfig = {}): Promise<void> {\n    if (this.isConsuming) {\n      this.logger.warn('Consumer already running');\n      return;\n    }\n\n    const mergedConfig = { ...DEFAULT_CONSUMER_CONFIG, ...config };\n\n    // Connect to RabbitMQ\n    await this.adapter.connect();\n\n    // Start consuming from operations queues (normal and priority variant)\n    const operationsQueueConfigs = [\n      { category: 'operations', variant: undefined },\n      { category: 'operations', variant: QueueVariant.PRIORITY },\n    ];\n\n    for (const queueConfig of operationsQueueConfigs) {\n      const queueName = QueueNamingUtils.buildQueueName(\n        this.applicationId,\n        queueConfig.category,\n        queueConfig.variant\n      );\n\n      // Create handler that matches MessageHandler signature\n      const handler: MessageHandler = async (msg: ConsumeMessage, channel: Channel) => {\n        // Phase 1 — each consumed job runs in its own correlation frame (fresh id). Phase 6 will seed\n        // it from the enqueuing request's originCorrelationId carried on the job payload.\n        await runInFreshRequestContextIfAbsent(() => this.handleMessage(msg, channel, mergedConfig));\n      };\n\n      await this.adapter.consume(queueName, handler);\n      this.consumingQueues.push(queueName);\n\n      this.logDebug('Started consuming from queue', {\n        queueName,\n        workerId: this.workerIdentity.workerId,\n      });\n    }\n\n    // Start consuming from scheduled queue (JWT-signed messages from crontabs-manager)\n    const scheduledQueueName = QueueNamingUtils.buildQueueName(this.applicationId, 'scheduled');\n\n    const scheduledHandler: MessageHandler = async (msg: ConsumeMessage, channel: Channel) => {\n      // Phase 1 — each cron tick (scheduled-queue message) runs in its own correlation frame.\n      await runInFreshRequestContextIfAbsent(() => this.handleScheduledMessage(msg, channel, mergedConfig));\n    };\n\n    await this.adapter.consume(scheduledQueueName, scheduledHandler);\n    this.consumingQueues.push(scheduledQueueName);\n\n    this.logDebug('Started consuming from scheduled queue', {\n      queueName: scheduledQueueName,\n      workerId: this.workerIdentity.workerId,\n    });\n\n    this.isConsuming = true;\n    this.logDebug('Job consumer started', {\n      workerId: this.workerIdentity.workerId,\n      queues: this.consumingQueues.length,\n    });\n  }\n\n  /**\n   * Stop consuming and close connections.\n   */\n  async stopConsuming(): Promise<void> {\n    if (!this.isConsuming) {\n      return;\n    }\n\n    // Cancel all consumers\n    for (const queueName of this.consumingQueues) {\n      await this.adapter.cancelConsumer(queueName);\n    }\n\n    this.consumingQueues = [];\n    this.isConsuming = false;\n\n    this.logDebug('Job consumer stopped', {\n      workerId: this.workerIdentity.workerId,\n    });\n  }\n\n  /**\n   * Handle a single message from the queue.\n   */\n  private async handleMessage(\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig\n  ): Promise<void> {\n    let job: JobPayload;\n\n    try {\n      job = JSON.parse(msg.content.toString());\n    } catch (parseError) {\n      this.logger.error('Failed to parse job message', {\n        ...projectCaughtErrorLogFields(parseError),\n      });\n      // Invalid message - don't requeue\n      channel.nack(msg, false, false);\n      return;\n    }\n\n    // Phase 6 — this job belongs to the trace of the request that enqueued it: adopt that request's\n    // originCorrelationId into the (freshly-minted) consume frame, so every log line + audit row this\n    // job produces on the worker carries the originating request's correlationId. Absent/malformed →\n    // the frame keeps its fresh id (a job with no originating request stays independently traceable).\n    adoptCorrelationIdIntoCurrentFrame(job.originCorrelationId);\n\n    const startTime = Date.now();\n    const { replyTo, correlationId } = msg.properties;\n\n    // Track in-flight job for graceful shutdown\n    this.shutdownManager.jobStarted();\n\n    this.logDebug('Processing job', {\n      jobId: job.jobId,\n      jobType: job.jobType,\n      priority: job.priority,\n      retryCount: job.retryCount ?? 0,\n    });\n\n    try {\n      // Mark job as started\n      await this.jobStatusTracking.markStarted(job.jobId, this.workerIdentity.workerId);\n\n      // Execute the job - returns JobResult which may indicate success OR failure\n      // The executor catches all errors and returns a result with status\n      const result = await this.jobExecutor.execute(job);\n\n      // Check if job failed (executor returns result with FAILED status, doesn't throw)\n      if (result.status === JobStatus.FAILED) {\n        // Convert the error result to proper retry handling\n        await this.handleJobFailure(job, result, msg, channel, config, startTime);\n        return;\n      }\n\n      // Mark as completed only on success\n      await this.jobStatusTracking.markCompleted(job.jobId);\n\n      // Send RPC reply if requested\n      if (replyTo && correlationId) {\n        await this.adapter.publishResult(replyTo, correlationId, result);\n      }\n\n      // Dispatch to completion handlers (for BACKGROUND_JOB callbacks)\n      await this.completionRegistry.dispatch({\n        result,\n        operationPathKey: job.operationPathKey,\n        batchRef: job.batchRef,\n      });\n\n      // Acknowledge success\n      channel.ack(msg);\n\n      this.logDebug('Job completed successfully', {\n        jobId: job.jobId,\n        durationMs: Date.now() - startTime,\n      });\n\n    } catch (error) {\n      // This catch block handles unexpected errors (e.g., from markStarted, dispatch)\n      // NOT job execution errors (those are returned by executor as FAILED status)\n      await this.handleJobError(job, error, msg, channel, config, startTime);\n    } finally {\n      // Always track job completion for graceful shutdown\n      this.shutdownManager.jobCompleted();\n    }\n  }\n\n  /**\n   * Handle job failure when executor returns FAILED status (not an exception).\n   * The executor catches errors and returns a JobResult with status FAILED.\n   */\n  private async handleJobFailure(\n    job: JobPayload,\n    result: JobResult,\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig,\n    startTime: number\n  ): Promise<void> {\n    const currentRetry = job.retryCount ?? 0;\n    const maxRetries = job.maxRetries ?? config.retryConfig?.maxRetries ?? QUEUE_RETRY_DEFAULTS.maxRetries;\n    const { replyTo, correlationId } = msg.properties;\n\n    // Extract error handling strategy from the error\n    const errorStrategy = result.error?.errorHandlingStrategy;\n    const shouldRetry = config.enableRetry &&\n      this.shouldRetryFromStrategy(errorStrategy, currentRetry, maxRetries);\n\n    if (shouldRetry) {\n      // Resolve backoff from the job's per-operation retryConfig, falling back to\n      // the consumer config then the framework defaults (per-field, so a partial\n      // authored retryConfig still gets sane values for the axes it omits).\n      const backoff = job.retryConfig;\n      const retryConfig: QueueRetryConfig = {\n        retryPolicy: backoff?.retryPolicy ?? config.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n        maxRetries: maxRetries,\n        baseDelayMs: backoff?.baseDelayMs ?? config.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n        maxDelayMs: backoff?.maxDelayMs ?? config.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n        timeoutMs: job.timeoutMs ?? config.retryConfig?.timeoutMs ?? QUEUE_RETRY_DEFAULTS.timeoutMs,\n      };\n\n      const delay = JobRetryUtils.calculateDelay(retryConfig, currentRetry);\n\n      // Create updated job with incremented retry count\n      const retryJob: JobPayload = {\n        ...job,\n        retryCount: currentRetry + 1,\n      };\n\n      // Durable backoff: hand the retry to the broker (delay queue holds it, then\n      // dead-letters it back) instead of an in-process setTimeout that would be\n      // lost if this worker crashes during the wait.\n      const operationsQueue = QueueNamingUtils.buildQueueName(this.applicationId, 'operations');\n      try {\n        await this.adapter.scheduleDelayedRetry(operationsQueue, retryJob, delay, {\n          replyTo,\n          correlationId,\n        });\n      } catch (scheduleError) {\n        // Could not hand the retry to the broker — requeue the ORIGINAL (do not\n        // ACK) so it is redelivered rather than silently lost.\n        this.logger.error('Failed to schedule durable retry; requeueing original', {\n          jobId: job.jobId,\n          ...projectCaughtErrorLogFields(scheduleError),\n        });\n        channel.nack(msg, false, true);\n        return;\n      }\n\n      // ACK original message (retry is now durably owned by the broker delay queue)\n      channel.ack(msg);\n\n      // Best-effort tracking update → RETRYING + lastRetryAt (never block the retry).\n      if (result.error) {\n        try {\n          await this.jobStatusTracking.markFailed(job.jobId, result.error);\n        } catch (trackingError) {\n          this.logDebug('Failed to mark job retrying (non-fatal)', {\n            jobId: job.jobId,\n            error: describeCaughtError(trackingError),\n          });\n        }\n      }\n\n      this.logger.warn('Job retry scheduled (durable delay queue)', {\n        jobId: job.jobId,\n        retryCount: currentRetry + 1,\n        maxRetries,\n        delayMs: delay,\n        // The queue is a non-HTTP boundary and owns its own operational level (a\n        // scheduled retry is a `warn`, not classified from the error's severity).\n        // Reuse the canonical field projection to restore the full error detail\n        // that no longer self-logs at construction.\n        ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n      });\n\n    } else {\n      // Max retries exceeded or non-retryable error\n      await this.jobStatusTracking.markDeadLettered(job.jobId);\n\n      // Send error reply if RPC\n      if (replyTo && correlationId) {\n        await this.adapter.publishResult(replyTo, correlationId, result);\n      }\n\n      // Reject without requeue → the operations queue's DLX now parks this\n      // permanently-failed job in `operations.deadletter` (previously the broker\n      // discarded it because no dead-letter-exchange was wired).\n      channel.nack(msg, false, false);\n\n      this.logger.error('Job failed permanently', {\n        jobId: job.jobId,\n        retryCount: currentRetry,\n        maxRetries,\n        // Permanent (dead-lettered) failure is operationally an `error`\n        // regardless of the underlying error's business severity, so the level\n        // stays explicit; the projection restores the full error detail (type,\n        // status, context) and the stack.\n        ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n        ...(result.error?.stack ? { stack: result.error.stack } : {}),\n      });\n    }\n  }\n\n  /**\n   * Check if retry should happen based on error strategy.\n   */\n  private shouldRetryFromStrategy(\n    strategy: ErrorHandling_Strategy | undefined,\n    currentRetry: number,\n    maxRetries: number\n  ): boolean {\n    // Check retry count limit\n    if (currentRetry >= maxRetries) {\n      return false;\n    }\n\n    // Only retry if strategy is explicitly RETRY or undefined (default to retry)\n    return strategy === undefined || strategy === ErrorHandling_Strategy.RETRY;\n  }\n\n  /**\n   * Handle unexpected errors (not job execution failures).\n   * This handles errors from status tracking, dispatch, etc.\n   */\n  private async handleJobError(\n    job: JobPayload,\n    error: unknown,\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig,\n    startTime: number\n  ): Promise<void> {\n    const currentRetry = job.retryCount ?? 0;\n    const maxRetries = job.maxRetries ?? config.retryConfig?.maxRetries ?? QUEUE_RETRY_DEFAULTS.maxRetries;\n    const { replyTo, correlationId } = msg.properties;\n\n    // This is a raw caught exception (not a returned JobResult). Normalize it\n    // once so both the retry and permanent-failure logs carry the full detail —\n    // errors no longer self-log at construction, so the queue boundary owns it.\n    const errorInfo = this.extractErrorInfo(error);\n    const errorLogFields: Record<string, unknown> = {\n      errorType: errorInfo.type,\n      ...(errorInfo.customMessageReference ? { customMessageReference: errorInfo.customMessageReference } : {}),\n      ...(errorInfo.statusCode ? { statusCode: errorInfo.statusCode } : {}),\n      ...(errorInfo.context ? { context: errorInfo.context } : {}),\n    };\n\n    // Check if we should retry\n    const shouldRetry = config.enableRetry && this.shouldRetry(error, currentRetry, maxRetries);\n\n    if (shouldRetry) {\n      // Resolve backoff from the job's per-operation retryConfig (per-field\n      // fallback to consumer config then framework defaults).\n      const backoff = job.retryConfig;\n      const retryConfig: QueueRetryConfig = {\n        retryPolicy: backoff?.retryPolicy ?? config.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n        maxRetries: maxRetries,\n        baseDelayMs: backoff?.baseDelayMs ?? config.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n        maxDelayMs: backoff?.maxDelayMs ?? config.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n        timeoutMs: job.timeoutMs ?? config.retryConfig?.timeoutMs ?? QUEUE_RETRY_DEFAULTS.timeoutMs,\n      };\n\n      const delay = JobRetryUtils.calculateDelay(retryConfig, currentRetry);\n\n      // Create updated job with incremented retry count\n      const retryJob: JobPayload = {\n        ...job,\n        retryCount: currentRetry + 1,\n      };\n\n      // Durable backoff via the broker delay queue (see handleJobFailure).\n      const operationsQueue = QueueNamingUtils.buildQueueName(this.applicationId, 'operations');\n      try {\n        await this.adapter.scheduleDelayedRetry(operationsQueue, retryJob, delay, {\n          replyTo,\n          correlationId,\n        });\n      } catch (scheduleError) {\n        this.logger.error('Failed to schedule durable retry; requeueing original', {\n          jobId: job.jobId,\n          ...projectCaughtErrorLogFields(scheduleError),\n        });\n        channel.nack(msg, false, true);\n        return;\n      }\n\n      // ACK original message (retry is now durably owned by the broker delay queue)\n      channel.ack(msg);\n\n      this.logger.warn('Job retry scheduled (durable delay queue)', {\n        jobId: job.jobId,\n        retryCount: currentRetry + 1,\n        maxRetries,\n        delayMs: delay,\n        ...errorLogFields,\n      });\n\n    } else {\n      // Max retries exceeded or non-retryable error\n\n      // Mark as dead-lettered\n      await this.jobStatusTracking.markDeadLettered(job.jobId);\n\n      // Send error reply if RPC\n      if (replyTo && correlationId) {\n        const jobResult: JobResult = {\n          jobId: job.jobId,\n          status: JobStatus.FAILED,\n          error: {\n            type: errorInfo.type ?? ErrorType.INTERNAL,\n            severity: ErrorSeverity.HIGH,\n            timestamp: new Date(),\n            context: errorInfo.context,\n            backendActions: [],\n            customMessageReference: ErrorCustomMessageReference.INTERNAL,\n            statusCode: 500,\n          },\n          startedAt: new Date(startTime),\n          completedAt: new Date(),\n          durationMs: Date.now() - startTime,\n        };\n\n        await this.adapter.publishResult(replyTo, correlationId, jobResult);\n      }\n\n      // Reject without requeue → the operations queue's DLX now parks this\n      // permanently-failed job in `operations.deadletter` (previously the broker\n      // discarded it because no dead-letter-exchange was wired).\n      channel.nack(msg, false, false);\n\n      this.logger.error('Job failed permanently', {\n        jobId: job.jobId,\n        retryCount: currentRetry,\n        maxRetries,\n        ...errorLogFields,\n      });\n    }\n  }\n\n  /**\n   * Determine if an error is retryable.\n   */\n  private shouldRetry(error: unknown, currentRetry: number, maxRetries: number): boolean {\n    // Check retry count limit\n    if (currentRetry >= maxRetries) {\n      return false;\n    }\n\n    // Check if error specifies non-retryable\n    if (this.isWildoError(error)) {\n      const strategy = error.errorHandlingStrategy;\n      // Only retry if strategy is explicitly RETRY or undefined (default to retry)\n      return strategy === undefined || strategy === ErrorHandling_Strategy.RETRY;\n    }\n\n    // Default: retry on any error\n    return true;\n  }\n\n  /**\n   * Type guard for WildoError_Backend.\n   */\n  private isWildoError(error: unknown): error is {\n    type: ErrorType;\n    errorHandlingStrategy?: ErrorHandling_Strategy;\n    customMessageReference?: ErrorCustomMessageReference;\n    statusCode?: number;\n    context?: Record<string, unknown>;\n  } {\n    return (\n      typeof error === 'object' &&\n      error !== null &&\n      'type' in error &&\n      typeof (error as any).type === 'string'\n    );\n  }\n\n  /**\n   * Extract error information for result/logging.\n   */\n  private extractErrorInfo(error: unknown): {\n    type?: ErrorType;\n    customMessageReference?: ErrorCustomMessageReference;\n    statusCode?: number;\n    context?: Record<string, unknown>;\n  } {\n    if (this.isWildoError(error)) {\n      return {\n        type: error.type,\n        customMessageReference: error.customMessageReference,\n        statusCode: error.statusCode,\n        context: error.context,\n      };\n    }\n\n    return {\n      type: ErrorType.INTERNAL,\n      context: {\n        message: describeCaughtError(error),\n      },\n    };\n  }\n\n  /**\n   * Handle a scheduled message (JWT-signed from crontabs-batches-manager).\n   */\n  private async handleScheduledMessage(\n    msg: ConsumeMessage,\n    channel: Channel,\n    config: JobConsumerConfig\n  ): Promise<void> {\n    const token = msg.content.toString();\n\n    // Track in-flight job for graceful shutdown\n    this.shutdownManager.jobStarted();\n\n    try {\n      // Verify the JWT token and convert to JobPayload\n      const job = await this.scheduledMessageVerifier.verifyAndConvert(token);\n\n      this.logger.info('Processing scheduled job', {\n        jobId: job.jobId,\n        hasBatchRef: !!job.batchRef,\n        hasOperationPathKey: !!job.operationPathKey,\n        applicationId: job.applicationId,\n      });\n\n      const startTime = Date.now();\n\n      // Mark job as started\n      await this.jobStatusTracking.markStarted(job.jobId, this.workerIdentity.workerId);\n\n      // Execute the job - returns JobResult which may indicate success OR failure\n      const result = await this.jobExecutor.execute(job);\n\n      // Check if job failed (executor returns result with FAILED status, doesn't throw)\n      if (result.status === JobStatus.FAILED) {\n        // Mark as failed in tracking\n        await this.jobStatusTracking.markFailed(job.jobId, result.error!);\n\n        // Log failure — the queue owns the operational level (`error`); the\n        // projection restores the full error detail that no longer self-logs at\n        // construction.\n        this.logger.error('Scheduled job execution failed', {\n          jobId: job.jobId,\n          durationMs: Date.now() - startTime,\n          ...(result.error ? projectWildoBackendErrorLogFields(result.error) : {}),\n          ...(result.error?.stack ? { stack: result.error.stack } : {}),\n        });\n\n        // Reject without requeue: scheduled jobs don't retry via the queue (they\n        // re-run on the next schedule). The message is now parked in\n        // `scheduled.deadletter` as a stale audit copy for inspection.\n        channel.nack(msg, false, false);\n        return;\n      }\n\n      // Mark as completed only on success\n      await this.jobStatusTracking.markCompleted(job.jobId);\n\n      // Acknowledge success\n      channel.ack(msg);\n\n      this.logger.info('Scheduled job completed successfully', {\n        jobId: job.jobId,\n        durationMs: Date.now() - startTime,\n      });\n\n    } catch (error) {\n      // This catches errors from token verification, status tracking, etc.\n      // NOT job execution errors (those are returned by executor as FAILED status)\n      this.logger.error('Scheduled job processing error', {\n        ...projectCaughtErrorLogFields(error),\n      });\n\n      // Reject without requeue: scheduled jobs don't retry via the queue (they\n      // re-run on the next schedule). The message is now parked in\n      // `scheduled.deadletter` as a stale audit copy for inspection.\n      channel.nack(msg, false, false);\n\n    } finally {\n      this.shutdownManager.jobCompleted();\n    }\n  }\n\n  /**\n   * Get worker identity for external access.\n   */\n  getWorkerIdentity(): WorkerIdentity {\n    return { ...this.workerIdentity };\n  }\n\n  /**\n   * Check if consumer is running.\n   */\n  isRunning(): boolean {\n    return this.isConsuming;\n  }\n}\n"]}

package/dist/esm/queue/job-executor.backend.service.d.ts

@@ -9,7 +9,7 @@ import { ErrorBuilderBackendService } from '../errors/services/error-builder.bac
 import { ServicesRegistryHandlerBackendService } from '../base-classes/services/services/services-registry-handler.backend.service';
 import { ExecutionContextCreatorBackendService } from '../runtime/services/execution-context-creator.backend.service';
 import { BatchesCronjobsExecutor_BackendService } from '../base-classes/services/batches/batches-cronjobs-executor.backend.service';
-import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';
+import type { QueueAdapter } from './adapters/queue-adapter.backend.interface';
 import { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';
 import { JobPayload, JobResult } from '@wildo-ai/saas-models';
 export declare class JobExecutorBackendService {
@@ -22,7 +22,7 @@ export declare class JobExecutorBackendService {
     private readonly errorBuilder;
     protected logDebug: (message: string, context?: Record<string, unknown>) => void;
     private readonly workerId;
-    constructor(servicesRegistry: ServicesRegistryHandlerBackendService, executionContextCreator: ExecutionContextCreatorBackendService, batchExecutor: BatchesCronjobsExecutor_BackendService, adapter: RabbitMQAdapterBackendService, payloadStorage: JobPayloadStorageBackendService, logger: LoggerService, errorBuilder: ErrorBuilderBackendService);
+    constructor(servicesRegistry: ServicesRegistryHandlerBackendService, executionContextCreator: ExecutionContextCreatorBackendService, batchExecutor: BatchesCronjobsExecutor_BackendService, adapter: QueueAdapter, payloadStorage: JobPayloadStorageBackendService, logger: LoggerService, errorBuilder: ErrorBuilderBackendService);
     /**
      * Execute a job and return result.
      */

package/dist/esm/queue/job-executor.backend.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"job-executor.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/job-executor.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,qCAAqC,EAAE,MAAM,6FAA6F,CAAC;AACpJ,OAAO,EAAE,qCAAqC,EAAE,MAAM,+EAA+E,CAAC;AACtI,OAAO,EAAE,sCAAsC,EAAE,MAAM,4FAA4F,CAAC;AACpJ,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EACL,UAAU,EACV,SAAS,EAQV,MAAM,uBAAuB,CAAC;AAgB/B,qBACa,yBAAyB;IAMlC,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAEjC,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IAExC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAE9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAExB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAE/B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAjB/B,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAIf,gBAAgB,EAAE,qCAAqC,EAEvD,uBAAuB,EAAE,qCAAqC,EAE9D,aAAa,EAAE,sCAAsC,EAErD,OAAO,EAAE,6BAA6B,EAEtC,cAAc,EAAE,+BAA+B,EAE/C,MAAM,EAAE,aAAa,EAErB,YAAY,EAAE,0BAA0B;IAO3D;;OAEG;IACU,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IAwEzD;;OAEG;YACW,wBAAwB;IA4BtC;;OAEG;YACW,kBAAkB;IAgBhC;;OAEG;YACW,oBAAoB;IAsBlC;;OAEG;IACI,WAAW,IAAI,MAAM;CAG7B"}
+{"version":3,"file":"job-executor.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/job-executor.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,qCAAqC,EAAE,MAAM,6FAA6F,CAAC;AACpJ,OAAO,EAAE,qCAAqC,EAAE,MAAM,+EAA+E,CAAC;AACtI,OAAO,EAAE,sCAAsC,EAAE,MAAM,4FAA4F,CAAC;AACpJ,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC/E,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EACL,UAAU,EACV,SAAS,EAQV,MAAM,uBAAuB,CAAC;AAgB/B,qBACa,yBAAyB;IAMlC,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAEjC,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IAExC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAE9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAExB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAE/B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAjB/B,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAIf,gBAAgB,EAAE,qCAAqC,EAEvD,uBAAuB,EAAE,qCAAqC,EAE9D,aAAa,EAAE,sCAAsC,EAErD,OAAO,EAAE,YAAY,EAErB,cAAc,EAAE,+BAA+B,EAE/C,MAAM,EAAE,aAAa,EAErB,YAAY,EAAE,0BAA0B;IAO3D;;OAEG;IACU,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IAwEzD;;OAEG;YACW,wBAAwB;IA4BtC;;OAEG;YACW,kBAAkB;IAgBhC;;OAEG;YACW,oBAAoB;IAsBlC;;OAEG;IACI,WAAW,IAAI,MAAM;CAG7B"}

package/dist/esm/queue/job-executor.backend.service.js

@@ -24,7 +24,6 @@ import { ErrorBuilderBackendService } from '../errors/services/error-builder.bac
 import { ServicesRegistryHandlerBackendService } from '../base-classes/services/services/services-registry-handler.backend.service.js';
 import { ExecutionContextCreatorBackendService } from '../runtime/services/execution-context-creator.backend.service.js';
 import { BatchesCronjobsExecutor_BackendService } from '../base-classes/services/batches/batches-cronjobs-executor.backend.service.js';
-import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service.js';
 import { JobPayloadStorageBackendService } from './job-payload-storage.backend.service.js';
 import { JobStatus, JobType, ErrorType, ErrorCustomMessageReference, ResourceSerializationUtils, ResourcesConfig_OperationPathUtils, } from '@wildo-ai/saas-models';
 /**
@@ -196,9 +195,7 @@ JobExecutorBackendService = __decorate([
     __param(6, inject(SAAS_SERVICE_TYPES.ErrorBuilderService)),
     __metadata("design:paramtypes", [ServicesRegistryHandlerBackendService,
         ExecutionContextCreatorBackendService,
-        BatchesCronjobsExecutor_BackendService,
-        RabbitMQAdapterBackendService,
-        JobPayloadStorageBackendService,
+        BatchesCronjobsExecutor_BackendService, Object, JobPayloadStorageBackendService,
         LoggerService,
         ErrorBuilderBackendService])
 ], JobExecutorBackendService);

package/dist/esm/queue/job-executor.backend.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"job-executor.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/job-executor.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,qCAAqC,EAAE,MAAM,6FAA6F,CAAC;AACpJ,OAAO,EAAE,qCAAqC,EAAE,MAAM,+EAA+E,CAAC;AACtI,OAAO,EAAE,sCAAsC,EAAE,MAAM,4FAA4F,CAAC;AACpJ,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAGL,SAAS,EACT,OAAO,EACP,SAAS,EACT,2BAA2B,EAC3B,0BAA0B,EAC1B,kCAAkC,GAEnC,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,MAAM,IAAI,KAAK;QACf,YAAY,IAAI,KAAK;QACrB,gBAAgB,IAAI,KAAK,CAC1B,CAAC;AACJ,CAAC;AAGM,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAMjB;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAjBT,QAAQ,CAA+D;IAChE,QAAQ,CAAS;IAElC,YAEmB,gBAAuD,EAEvD,uBAA8D,EAE9D,aAAqD,EAErD,OAAsC,EAEtC,cAA+C,EAE/C,MAAqB,EAErB,YAAwC;QAZxC,qBAAgB,GAAhB,gBAAgB,CAAuC;QAEvD,4BAAuB,GAAvB,uBAAuB,CAAuC;QAE9D,kBAAa,GAAb,aAAa,CAAwC;QAErD,YAAO,GAAP,OAAO,CAA+B;QAEtC,mBAAc,GAAd,cAAc,CAAiC;QAE/C,WAAM,GAAN,MAAM,CAAe;QAErB,iBAAY,GAAZ,YAAY,CAA4B;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QACpE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC5D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,OAAO,CAAC,GAAe;QAClC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,wDAAwD;YACxD,IAAI,SAAkB,CAAC;YACvB,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;gBAC9B,iCAAiC;gBACjC,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBAC1E,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;oBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,SAAS,EAAE,GAAG,CAAC,qBAAqB;iBACrC,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC7B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC5C,CAAC;YAED,4BAA4B;YAC5B,IAAI,MAAW,CAAC;YAEhB,QAAQ,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpB,KAAK,OAAO,CAAC,kBAAkB;oBAC7B,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBAC7D,MAAM;gBACR,KAAK,OAAO,CAAC,YAAY;oBACvB,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACvD,MAAM;gBACR,KAAK,OAAO,CAAC,cAAc;oBACzB,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACzD,MAAM;gBACR;oBACE,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,QAAQ,EAClB,SAAS,EACT;wBACE,sBAAsB,EAAE,2BAA2B,CAAC,oBAAoB;wBACxE,OAAO,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE;qBAC/D,CACF,CAAC;YACN,CAAC;YAED,uBAAuB;YACvB,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,SAAS,CAAC,SAAS;gBAC3B,UAAU,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;gBACrE,SAAS;gBACT,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;aAC7C,CAAC;YAEF,OAAO,SAAS,CAAC;QAEnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wBAAwB;YACxB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC;gBACpC,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,KAAc,CAAC,CAAC;YAE7D,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,UAAU;gBACjB,SAAS;gBACT,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;aAC7C,CAAC;YAEF,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,GAAe,EAAE,SAAkB;QACxE,IAAI,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACxD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;gBACE,sBAAsB,EAAE,2BAA2B,CAAC,6BAA6B;gBACjF,OAAO,EAAE,EAAE,OAAO,EAAE,wEAAwE,EAAE;aAC/F,CACF,CAAC;QACJ,CAAC;QAED,qDAAqD;QACrD,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,6BAA6B,CACvF,GAAG,CAAC,oBAAoB,EACxB,GAAG,CAAC,iBAAiB,CACtB,CAAC;QACF,MAAM,aAAa,GAAG,0BAA0B,CAAC,wBAAwB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAEjG,yEAAyE;QACzE,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE;YACtG,iBAAiB,EAAE,IAAI;YACvB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,wCAAwC,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAAC,GAAe,EAAE,SAAkB;QAClE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;gBACE,sBAAsB,EAAE,2BAA2B,CAAC,uBAAuB;gBAC3E,OAAO,EAAE,EAAE,OAAO,EAAE,oCAAoC,EAAE;aAC3D,CACF,CAAC;QACJ,CAAC;QAED,yDAAyD;QACzD,OAAO,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,GAAe,EAAE,SAAkB;QACpE,mEAAmE;QACnE,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACzB,kCAAkC;YAClC,+FAA+F;YAC/F,MAAM,aAAa,GAAG,kCAAkC,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC,aAAa,CAAC,wBAAwB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC/E,CAAC;aAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACxB,eAAe;YACf,OAAO,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;YACE,sBAAsB,EAAE,2BAA2B,CAAC,oBAAoB;YACxE,OAAO,EAAE,EAAE,OAAO,EAAE,yDAAyD,EAAE;SAChF,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,WAAW;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF,CAAA;AArLY,yBAAyB;IADrC,UAAU,EAAE;IAMR,WAAA,MAAM,CAAC,kBAAkB,CAAC,8BAA8B,CAAC,CAAA;IAEzD,WAAA,MAAM,CAAC,kBAAkB,CAAC,qCAAqC,CAAC,CAAA;IAEhE,WAAA,MAAM,CAAC,kBAAkB,CAAC,8BAA8B,CAAC,CAAA;IAEzD,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAA;qCAXZ,qCAAqC;QAE9B,qCAAqC;QAE/C,sCAAsC;QAE5C,6BAA6B;QAEtB,+BAA+B;QAEvC,aAAa;QAEP,0BAA0B;GAlBhD,yBAAyB,CAqLrC","sourcesContent":["/**\n * Job Executor Backend Service\n *\n * Executes queued jobs on worker pods.\n * Handles resource operations, custom batches, and scheduled crons.\n */\n\nimport { injectable, inject } from 'inversify';\nimport * as os from 'os';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';\nimport { ServicesRegistryHandlerBackendService } from '~wildo-backend-lib/base-classes/services/services/services-registry-handler.backend.service';\nimport { ExecutionContextCreatorBackendService } from '~wildo-backend-lib/runtime/services/execution-context-creator.backend.service';\nimport { BatchesCronjobsExecutor_BackendService } from '~wildo-backend-lib/base-classes/services/batches/batches-cronjobs-executor.backend.service';\nimport { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';\nimport { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport {\n  JobPayload,\n  JobResult,\n  JobStatus,\n  JobType,\n  ErrorType,\n  ErrorCustomMessageReference,\n  ResourceSerializationUtils,\n  ResourcesConfig_OperationPathUtils,\n  WildoError_Backend,\n} from '@wildo-ai/saas-models';\n\n/**\n * Type guard to check if an error is a WildoError_Backend.\n * Uses duck typing since WildoError_Backend is a type, not a class.\n */\nfunction isWildoError(error: unknown): error is WildoError_Backend {\n  return (\n    error !== null &&\n    typeof error === 'object' &&\n    'type' in error &&\n    'statusCode' in error &&\n    'backendActions' in error\n  );\n}\n\n@injectable()\nexport class JobExecutorBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private readonly workerId: string;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.ServicesRegistryHandlerService)\n    private readonly servicesRegistry: ServicesRegistryHandlerBackendService,\n    @inject(SAAS_SERVICE_TYPES.ExecutionContextCreatorBackendService)\n    private readonly executionContextCreator: ExecutionContextCreatorBackendService,\n    @inject(SAAS_SERVICE_TYPES.BatchesCronjobsExecutorService)\n    private readonly batchExecutor: BatchesCronjobsExecutor_BackendService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly adapter: RabbitMQAdapterBackendService,\n    @inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)\n    private readonly payloadStorage: JobPayloadStorageBackendService,\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)\n    private readonly errorBuilder: ErrorBuilderBackendService,\n  ) {\n    this.logDebug = this.logger.debugByType('JobExecutorBackendService');\n    this.logger.setDebugTypeLogging('JobExecutorBackendService', false);\n    this.workerId = process.env.K8S_POD_NAME || os.hostname();\n  }\n\n  /**\n   * Execute a job and return result.\n   */\n  public async execute(job: JobPayload): Promise<JobResult> {\n    const startedAt = new Date();\n\n    try {\n      // Resolve input data - check if stored externally first\n      let inputData: unknown;\n      if (job.inputPayloadStorageId) {\n        // Retrieve from external storage\n        inputData = await this.payloadStorage.retrieve(job.inputPayloadStorageId);\n        this.logDebug('Retrieved external payload', {\n          jobId: job.jobId,\n          storageId: job.inputPayloadStorageId,\n        });\n      } else if (job.inputDataJson) {\n        inputData = JSON.parse(job.inputDataJson);\n      }\n\n      // Execute based on job type\n      let result: any;\n\n      switch (job.jobType) {\n        case JobType.RESOURCE_OPERATION:\n          result = await this.executeResourceOperation(job, inputData);\n          break;\n        case JobType.CUSTOM_BATCH:\n          result = await this.executeCustomBatch(job, inputData);\n          break;\n        case JobType.SCHEDULED_CRON:\n          result = await this.executeScheduledCron(job, inputData);\n          break;\n        default:\n          throw this.errorBuilder.buildError(\n            ErrorType.INTERNAL,\n            undefined,\n            {\n              customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,\n              context: { jobType: job.jobType, message: 'Unknown job type' },\n            }\n          );\n      }\n\n      // Build success result\n      const jobResult: JobResult = {\n        jobId: job.jobId,\n        status: JobStatus.COMPLETED,\n        resultJson: result !== undefined ? JSON.stringify(result) : undefined,\n        startedAt,\n        completedAt: new Date(),\n        durationMs: Date.now() - startedAt.getTime(),\n      };\n\n      return jobResult;\n\n    } catch (error) {\n      // Convert to WildoError\n      const wildoError = isWildoError(error)\n        ? error\n        : this.errorBuilder.buildFromStandardError(error as Error);\n\n      const jobResult: JobResult = {\n        jobId: job.jobId,\n        status: JobStatus.FAILED,\n        error: wildoError,\n        startedAt,\n        completedAt: new Date(),\n        durationMs: Date.now() - startedAt.getTime(),\n      };\n\n      return jobResult;\n    }\n  }\n\n  /**\n   * Execute resource operation with reconstructed execution context.\n   */\n  private async executeResourceOperation(job: JobPayload, inputData: unknown): Promise<any> {\n    if (!job.executionContextJson || !job.operationPathJson) {\n      throw this.errorBuilder.buildError(\n        ErrorType.CONFIGURATION,\n        undefined,\n        {\n          customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_CONTEXT_INVALID,\n          context: { message: 'Resource operation requires executionContextJson and operationPathJson' },\n        }\n      );\n    }\n\n    // Reconstruct execution context from serialized data\n    const executionContext = await this.executionContextCreator.reconstructFromSerializedData(\n      job.executionContextJson,\n      job.operationPathJson\n    );\n    const operationPath = ResourceSerializationUtils.deserializeOperationPath(job.operationPathJson);\n\n    // Execute by the serialized operation path and preserve the enqueue-time\n    // target id. Context-addressed routes are resolved and authorized before\n    // enqueueing; worker replay must not re-derive identity from context.\n    return this.servicesRegistry.doByOperationPath(operationPath, executionContext, job.inputId, inputData, {\n      isWorkerExecution: true,\n      ...(job.inputId ? { contextAddressedOperationAuthorizationId: job.inputId } : {}),\n    });\n  }\n\n  /**\n   * Execute custom batch job.\n   */\n  private async executeCustomBatch(job: JobPayload, inputData: unknown): Promise<any> {\n    if (!job.batchRef) {\n      throw this.errorBuilder.buildError(\n        ErrorType.CONFIGURATION,\n        undefined,\n        {\n          customMessageReference: ErrorCustomMessageReference.JOB_BATCH_REF_NOT_FOUND,\n          context: { message: 'Custom batch job requires batchRef' },\n        }\n      );\n    }\n\n    // Custom batch executor creates its own internal context\n    return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());\n  }\n\n  /**\n   * Execute scheduled cron job (after JWT verification in consumer).\n   */\n  private async executeScheduledCron(job: JobPayload, inputData: unknown): Promise<any> {\n    // Scheduled jobs can be either resource-oriented or custom batches\n    if (job.operationPathKey) {\n      // Resource-oriented scheduled job\n      // operationPathKey is in format: \"resource | operation | variantType | variant | isBulk:false\"\n      const operationPath = ResourcesConfig_OperationPathUtils.parseOperationPathKey(job.operationPathKey);\n      return this.batchExecutor.executeBatchFromResource(operationPath, inputData);\n    } else if (job.batchRef) {\n      // Custom batch\n      return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());\n    }\n\n    throw this.errorBuilder.buildError(\n      ErrorType.CONFIGURATION,\n      undefined,\n      {\n        customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,\n        context: { message: 'Scheduled job has neither operationPathKey nor batchRef' },\n      }\n    );\n  }\n\n  /**\n   * Get worker ID for this executor.\n   */\n  public getWorkerId(): string {\n    return this.workerId;\n  }\n}\n"]}
+{"version":3,"file":"job-executor.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/job-executor.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,0BAA0B,EAAE,MAAM,kEAAkE,CAAC;AAC9G,OAAO,EAAE,qCAAqC,EAAE,MAAM,6FAA6F,CAAC;AACpJ,OAAO,EAAE,qCAAqC,EAAE,MAAM,+EAA+E,CAAC;AACtI,OAAO,EAAE,sCAAsC,EAAE,MAAM,4FAA4F,CAAC;AAEpJ,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAGL,SAAS,EACT,OAAO,EACP,SAAS,EACT,2BAA2B,EAC3B,0BAA0B,EAC1B,kCAAkC,GAEnC,MAAM,uBAAuB,CAAC;AAE/B;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,MAAM,IAAI,KAAK;QACf,YAAY,IAAI,KAAK;QACrB,gBAAgB,IAAI,KAAK,CAC1B,CAAC;AACJ,CAAC;AAGM,IAAM,yBAAyB,GAA/B,MAAM,yBAAyB;IAMjB;IAEA;IAEA;IAEA;IAEA;IAEA;IAEA;IAjBT,QAAQ,CAA+D;IAChE,QAAQ,CAAS;IAElC,YAEmB,gBAAuD,EAEvD,uBAA8D,EAE9D,aAAqD,EAErD,OAAqB,EAErB,cAA+C,EAE/C,MAAqB,EAErB,YAAwC;QAZxC,qBAAgB,GAAhB,gBAAgB,CAAuC;QAEvD,4BAAuB,GAAvB,uBAAuB,CAAuC;QAE9D,kBAAa,GAAb,aAAa,CAAwC;QAErD,YAAO,GAAP,OAAO,CAAc;QAErB,mBAAc,GAAd,cAAc,CAAiC;QAE/C,WAAM,GAAN,MAAM,CAAe;QAErB,iBAAY,GAAZ,YAAY,CAA4B;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QACpE,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC5D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,OAAO,CAAC,GAAe;QAClC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,wDAAwD;YACxD,IAAI,SAAkB,CAAC;YACvB,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;gBAC9B,iCAAiC;gBACjC,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBAC1E,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;oBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,SAAS,EAAE,GAAG,CAAC,qBAAqB;iBACrC,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC7B,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC5C,CAAC;YAED,4BAA4B;YAC5B,IAAI,MAAW,CAAC;YAEhB,QAAQ,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpB,KAAK,OAAO,CAAC,kBAAkB;oBAC7B,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBAC7D,MAAM;gBACR,KAAK,OAAO,CAAC,YAAY;oBACvB,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACvD,MAAM;gBACR,KAAK,OAAO,CAAC,cAAc;oBACzB,MAAM,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACzD,MAAM;gBACR;oBACE,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,QAAQ,EAClB,SAAS,EACT;wBACE,sBAAsB,EAAE,2BAA2B,CAAC,oBAAoB;wBACxE,OAAO,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE;qBAC/D,CACF,CAAC;YACN,CAAC;YAED,uBAAuB;YACvB,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,SAAS,CAAC,SAAS;gBAC3B,UAAU,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;gBACrE,SAAS;gBACT,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;aAC7C,CAAC;YAEF,OAAO,SAAS,CAAC;QAEnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wBAAwB;YACxB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC;gBACpC,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,sBAAsB,CAAC,KAAc,CAAC,CAAC;YAE7D,MAAM,SAAS,GAAc;gBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,UAAU;gBACjB,SAAS;gBACT,WAAW,EAAE,IAAI,IAAI,EAAE;gBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;aAC7C,CAAC;YAEF,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,GAAe,EAAE,SAAkB;QACxE,IAAI,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;YACxD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;gBACE,sBAAsB,EAAE,2BAA2B,CAAC,6BAA6B;gBACjF,OAAO,EAAE,EAAE,OAAO,EAAE,wEAAwE,EAAE;aAC/F,CACF,CAAC;QACJ,CAAC;QAED,qDAAqD;QACrD,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,6BAA6B,CACvF,GAAG,CAAC,oBAAoB,EACxB,GAAG,CAAC,iBAAiB,CACtB,CAAC;QACF,MAAM,aAAa,GAAG,0BAA0B,CAAC,wBAAwB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAEjG,yEAAyE;QACzE,yEAAyE;QACzE,sEAAsE;QACtE,OAAO,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE;YACtG,iBAAiB,EAAE,IAAI;YACvB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,wCAAwC,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAAC,GAAe,EAAE,SAAkB;QAClE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;gBACE,sBAAsB,EAAE,2BAA2B,CAAC,uBAAuB;gBAC3E,OAAO,EAAE,EAAE,OAAO,EAAE,oCAAoC,EAAE;aAC3D,CACF,CAAC;QACJ,CAAC;QAED,yDAAyD;QACzD,OAAO,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,GAAe,EAAE,SAAkB;QACpE,mEAAmE;QACnE,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACzB,kCAAkC;YAClC,+FAA+F;YAC/F,MAAM,aAAa,GAAG,kCAAkC,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC,aAAa,CAAC,wBAAwB,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC/E,CAAC;aAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACxB,eAAe;YACf,OAAO,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAChC,SAAS,CAAC,aAAa,EACvB,SAAS,EACT;YACE,sBAAsB,EAAE,2BAA2B,CAAC,oBAAoB;YACxE,OAAO,EAAE,EAAE,OAAO,EAAE,yDAAyD,EAAE;SAChF,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,WAAW;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;CACF,CAAA;AArLY,yBAAyB;IADrC,UAAU,EAAE;IAMR,WAAA,MAAM,CAAC,kBAAkB,CAAC,8BAA8B,CAAC,CAAA;IAEzD,WAAA,MAAM,CAAC,kBAAkB,CAAC,qCAAqC,CAAC,CAAA;IAEhE,WAAA,MAAM,CAAC,kBAAkB,CAAC,8BAA8B,CAAC,CAAA;IAEzD,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAA;qCAXZ,qCAAqC;QAE9B,qCAAqC;QAE/C,sCAAsC,UAIrC,+BAA+B;QAEvC,aAAa;QAEP,0BAA0B;GAlBhD,yBAAyB,CAqLrC","sourcesContent":["/**\n * Job Executor Backend Service\n *\n * Executes queued jobs on worker pods.\n * Handles resource operations, custom batches, and scheduled crons.\n */\n\nimport { injectable, inject } from 'inversify';\nimport * as os from 'os';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';\nimport { ServicesRegistryHandlerBackendService } from '~wildo-backend-lib/base-classes/services/services/services-registry-handler.backend.service';\nimport { ExecutionContextCreatorBackendService } from '~wildo-backend-lib/runtime/services/execution-context-creator.backend.service';\nimport { BatchesCronjobsExecutor_BackendService } from '~wildo-backend-lib/base-classes/services/batches/batches-cronjobs-executor.backend.service';\nimport type { QueueAdapter } from './adapters/queue-adapter.backend.interface';\nimport { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport {\n  JobPayload,\n  JobResult,\n  JobStatus,\n  JobType,\n  ErrorType,\n  ErrorCustomMessageReference,\n  ResourceSerializationUtils,\n  ResourcesConfig_OperationPathUtils,\n  WildoError_Backend,\n} from '@wildo-ai/saas-models';\n\n/**\n * Type guard to check if an error is a WildoError_Backend.\n * Uses duck typing since WildoError_Backend is a type, not a class.\n */\nfunction isWildoError(error: unknown): error is WildoError_Backend {\n  return (\n    error !== null &&\n    typeof error === 'object' &&\n    'type' in error &&\n    'statusCode' in error &&\n    'backendActions' in error\n  );\n}\n\n@injectable()\nexport class JobExecutorBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private readonly workerId: string;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.ServicesRegistryHandlerService)\n    private readonly servicesRegistry: ServicesRegistryHandlerBackendService,\n    @inject(SAAS_SERVICE_TYPES.ExecutionContextCreatorBackendService)\n    private readonly executionContextCreator: ExecutionContextCreatorBackendService,\n    @inject(SAAS_SERVICE_TYPES.BatchesCronjobsExecutorService)\n    private readonly batchExecutor: BatchesCronjobsExecutor_BackendService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly adapter: QueueAdapter,\n    @inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)\n    private readonly payloadStorage: JobPayloadStorageBackendService,\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)\n    private readonly errorBuilder: ErrorBuilderBackendService,\n  ) {\n    this.logDebug = this.logger.debugByType('JobExecutorBackendService');\n    this.logger.setDebugTypeLogging('JobExecutorBackendService', false);\n    this.workerId = process.env.K8S_POD_NAME || os.hostname();\n  }\n\n  /**\n   * Execute a job and return result.\n   */\n  public async execute(job: JobPayload): Promise<JobResult> {\n    const startedAt = new Date();\n\n    try {\n      // Resolve input data - check if stored externally first\n      let inputData: unknown;\n      if (job.inputPayloadStorageId) {\n        // Retrieve from external storage\n        inputData = await this.payloadStorage.retrieve(job.inputPayloadStorageId);\n        this.logDebug('Retrieved external payload', {\n          jobId: job.jobId,\n          storageId: job.inputPayloadStorageId,\n        });\n      } else if (job.inputDataJson) {\n        inputData = JSON.parse(job.inputDataJson);\n      }\n\n      // Execute based on job type\n      let result: any;\n\n      switch (job.jobType) {\n        case JobType.RESOURCE_OPERATION:\n          result = await this.executeResourceOperation(job, inputData);\n          break;\n        case JobType.CUSTOM_BATCH:\n          result = await this.executeCustomBatch(job, inputData);\n          break;\n        case JobType.SCHEDULED_CRON:\n          result = await this.executeScheduledCron(job, inputData);\n          break;\n        default:\n          throw this.errorBuilder.buildError(\n            ErrorType.INTERNAL,\n            undefined,\n            {\n              customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,\n              context: { jobType: job.jobType, message: 'Unknown job type' },\n            }\n          );\n      }\n\n      // Build success result\n      const jobResult: JobResult = {\n        jobId: job.jobId,\n        status: JobStatus.COMPLETED,\n        resultJson: result !== undefined ? JSON.stringify(result) : undefined,\n        startedAt,\n        completedAt: new Date(),\n        durationMs: Date.now() - startedAt.getTime(),\n      };\n\n      return jobResult;\n\n    } catch (error) {\n      // Convert to WildoError\n      const wildoError = isWildoError(error)\n        ? error\n        : this.errorBuilder.buildFromStandardError(error as Error);\n\n      const jobResult: JobResult = {\n        jobId: job.jobId,\n        status: JobStatus.FAILED,\n        error: wildoError,\n        startedAt,\n        completedAt: new Date(),\n        durationMs: Date.now() - startedAt.getTime(),\n      };\n\n      return jobResult;\n    }\n  }\n\n  /**\n   * Execute resource operation with reconstructed execution context.\n   */\n  private async executeResourceOperation(job: JobPayload, inputData: unknown): Promise<any> {\n    if (!job.executionContextJson || !job.operationPathJson) {\n      throw this.errorBuilder.buildError(\n        ErrorType.CONFIGURATION,\n        undefined,\n        {\n          customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_CONTEXT_INVALID,\n          context: { message: 'Resource operation requires executionContextJson and operationPathJson' },\n        }\n      );\n    }\n\n    // Reconstruct execution context from serialized data\n    const executionContext = await this.executionContextCreator.reconstructFromSerializedData(\n      job.executionContextJson,\n      job.operationPathJson\n    );\n    const operationPath = ResourceSerializationUtils.deserializeOperationPath(job.operationPathJson);\n\n    // Execute by the serialized operation path and preserve the enqueue-time\n    // target id. Context-addressed routes are resolved and authorized before\n    // enqueueing; worker replay must not re-derive identity from context.\n    return this.servicesRegistry.doByOperationPath(operationPath, executionContext, job.inputId, inputData, {\n      isWorkerExecution: true,\n      ...(job.inputId ? { contextAddressedOperationAuthorizationId: job.inputId } : {}),\n    });\n  }\n\n  /**\n   * Execute custom batch job.\n   */\n  private async executeCustomBatch(job: JobPayload, inputData: unknown): Promise<any> {\n    if (!job.batchRef) {\n      throw this.errorBuilder.buildError(\n        ErrorType.CONFIGURATION,\n        undefined,\n        {\n          customMessageReference: ErrorCustomMessageReference.JOB_BATCH_REF_NOT_FOUND,\n          context: { message: 'Custom batch job requires batchRef' },\n        }\n      );\n    }\n\n    // Custom batch executor creates its own internal context\n    return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());\n  }\n\n  /**\n   * Execute scheduled cron job (after JWT verification in consumer).\n   */\n  private async executeScheduledCron(job: JobPayload, inputData: unknown): Promise<any> {\n    // Scheduled jobs can be either resource-oriented or custom batches\n    if (job.operationPathKey) {\n      // Resource-oriented scheduled job\n      // operationPathKey is in format: \"resource | operation | variantType | variant | isBulk:false\"\n      const operationPath = ResourcesConfig_OperationPathUtils.parseOperationPathKey(job.operationPathKey);\n      return this.batchExecutor.executeBatchFromResource(operationPath, inputData);\n    } else if (job.batchRef) {\n      // Custom batch\n      return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());\n    }\n\n    throw this.errorBuilder.buildError(\n      ErrorType.CONFIGURATION,\n      undefined,\n      {\n        customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,\n        context: { message: 'Scheduled job has neither operationPathKey nor batchRef' },\n      }\n    );\n  }\n\n  /**\n   * Get worker ID for this executor.\n   */\n  public getWorkerId(): string {\n    return this.workerId;\n  }\n}\n"]}

package/dist/esm/queue/queue-monitoring.backend.service.d.ts

@@ -6,7 +6,7 @@
  */
 import { LoggerService } from '../monitoring/logger/services/logger.service';
 import { AppConfigurationService } from '../config/services/app-configuration.backend.service';
-import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';
+import type { QueueAdapter } from './adapters/queue-adapter.backend.interface';
 import { QueueMetrics, QueueHealthSummary, QueueAlertThreshold } from '@wildo-ai/saas-models';
 /**
  * Alert callback type for queue monitoring.
@@ -15,13 +15,13 @@ export type QueueAlertCallback = (alertType: string, queueName: string, details:
 export declare class QueueMonitoringBackendService {
     private readonly logger;
     private readonly configService;
-    private readonly rabbitMQAdapter;
+    private readonly queueAdapter;
     protected logDebug: (message: string, context?: Record<string, unknown>) => void;
     private alertCallbacks;
     private alertThresholds;
     private monitoringInterval;
     private isMonitoring;
-    constructor(logger: LoggerService, configService: AppConfigurationService, rabbitMQAdapter: RabbitMQAdapterBackendService);
+    constructor(logger: LoggerService, configService: AppConfigurationService, queueAdapter: QueueAdapter);
     /**
      * Start monitoring queues at the specified interval.
      */
@@ -48,7 +48,11 @@ export declare class QueueMonitoringBackendService {
      */
     getHealthSummary(): Promise<QueueHealthSummary>;
     /**
-     * Check if RabbitMQ connection is healthy.
+     * Whether the queue substrate this runtime uses is connected.
+     *
+     * Named `isRabbitMQHealthy` for its callers' sake and answered by whichever adapter is bound
+     * (#502) — an in-memory queue reports connected once `connect()` has run, which is the honest
+     * answer to "can this runtime enqueue work" regardless of what is behind it.
      */
     isRabbitMQHealthy(): Promise<boolean>;
     /**

package/dist/esm/queue/queue-monitoring.backend.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"queue-monitoring.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/queue-monitoring.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAE7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EACL,YAAY,EACZ,kBAAkB,EAElB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC7B,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B,qBACa,6BAA6B;IAStC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAE9B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAZlC,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,kBAAkB,CAA+B;IACzD,OAAO,CAAC,YAAY,CAAS;gBAIV,MAAM,EAAE,aAAa,EAErB,aAAa,EAAE,uBAAuB,EAEtC,eAAe,EAAE,6BAA6B;IAMjE;;OAEG;IACH,eAAe,CAAC,UAAU,GAAE,MAAc,GAAG,IAAI;IA2BjD;;OAEG;IACH,cAAc,IAAI,IAAI;IAStB;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI;IAI3C;;OAEG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,GAAG,IAAI;IAK1E;;;OAGG;IACG,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAwC/D;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAsErD;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAI3C;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAsC3B;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,YAAY;IAkB1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAgB3B"}
+{"version":3,"file":"queue-monitoring.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/queue-monitoring.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAE7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC/E,OAAO,EACL,YAAY,EACZ,kBAAkB,EAElB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC7B,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE1B,qBACa,6BAA6B;IAStC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAE9B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAZ/B,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,kBAAkB,CAA+B;IACzD,OAAO,CAAC,YAAY,CAAS;gBAIV,MAAM,EAAE,aAAa,EAErB,aAAa,EAAE,uBAAuB,EAEtC,YAAY,EAAE,YAAY;IAM7C;;OAEG;IACH,eAAe,CAAC,UAAU,GAAE,MAAc,GAAG,IAAI;IA2BjD;;OAEG;IACH,cAAc,IAAI,IAAI;IAStB;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI;IAI3C;;OAEG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,GAAG,IAAI;IAK1E;;;OAGG;IACG,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA4C/D;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAsErD;;;;;;OAMG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAI3C;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAsC3B;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,YAAY;IAkB1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAgB3B"}

package/dist/esm/queue/queue-monitoring.backend.service.js

@@ -21,21 +21,20 @@ import { SAAS_SERVICE_TYPES } from '../inject/models/service-types.js';
 import { LoggerService } from '../monitoring/logger/services/logger.service.js';
 import { describeCaughtError, projectCaughtErrorLogFields } from '../errors/utils/error-log-level-classification.backend.utils.js';
 import { AppConfigurationService } from '../config/services/app-configuration.backend.service.js';
-import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service.js';
 import { QueueHealthStatus, } from '@wildo-ai/saas-models';
 let QueueMonitoringBackendService = class QueueMonitoringBackendService {
     logger;
     configService;
-    rabbitMQAdapter;
+    queueAdapter;
     logDebug;
     alertCallbacks = [];
     alertThresholds = new Map();
     monitoringInterval = null;
     isMonitoring = false;
-    constructor(logger, configService, rabbitMQAdapter) {
+    constructor(logger, configService, queueAdapter) {
         this.logger = logger;
         this.configService = configService;
-        this.rabbitMQAdapter = rabbitMQAdapter;
+        this.queueAdapter = queueAdapter;
         this.logDebug = this.logger.debugByType('QueueMonitoringBackendService');
         this.logger.setDebugTypeLogging('QueueMonitoringBackendService', false);
     }
@@ -95,20 +94,26 @@ let QueueMonitoringBackendService = class QueueMonitoringBackendService {
      * Also checks the corresponding dead letter queue if it exists.
      */
     async getQueueMetrics(queueName) {
-        const channel = this.rabbitMQAdapter.getChannel();
-        if (!channel) {
+        /*
+         * Asks the ADAPTER how deep the queue is, rather than reaching through it for an amqplib
+         * channel (#502). Behaviour is unchanged for RabbitMQ — `null` here is what a null channel
+         * produced before, and it still yields empty metrics — but the question is now one every
+         * adapter can answer about itself, so a broker-free runtime reports real numbers instead of
+         * being structurally unable to be monitored.
+         *
+         * Dead-letter depth stays unreported: on RabbitMQ `checkQueue` CLOSES the channel when the
+         * queue does not exist, and only specific DLQs exist (`operations.deadletter`, not the
+         * per-variant `operations.priority.deadletter`). That needs an explicit queue-name list, and it
+         * is unrelated to this change.
+         */
+        const pendingCount = await this.queueAdapter.getQueueDepth(queueName);
+        if (pendingCount === null) {
             return this.createEmptyMetrics(queueName);
         }
         try {
-            // Get queue info from RabbitMQ
-            const queueInfo = await channel.checkQueue(queueName);
-            // Note: Dead letter queue check is skipped because channel.checkQueue() closes the channel
-            // if the queue doesn't exist. Only specific DLQ queues exist (operations.deadletter, etc.),
-            // not per-variant (operations.priority.deadletter). DLQ monitoring should be done via
-            // explicit queue name list instead.
             return {
                 queueName,
-                pendingCount: queueInfo.messageCount,
+                pendingCount,
                 processingCount: 0, // Would need consumer tracking
                 deadLetterCount: 0, // Skipped - see note above
                 publishRatePerMinute: 0, // Would need historical tracking
@@ -195,10 +200,14 @@ let QueueMonitoringBackendService = class QueueMonitoringBackendService {
         };
     }
     /**
-     * Check if RabbitMQ connection is healthy.
+     * Whether the queue substrate this runtime uses is connected.
+     *
+     * Named `isRabbitMQHealthy` for its callers' sake and answered by whichever adapter is bound
+     * (#502) — an in-memory queue reports connected once `connect()` has run, which is the honest
+     * answer to "can this runtime enqueue work" regardless of what is behind it.
      */
     async isRabbitMQHealthy() {
-        return this.rabbitMQAdapter.isConnected();
+        return this.queueAdapter.isConnected();
     }
     /**
      * Evaluate queue health based on thresholds.
@@ -291,8 +300,7 @@ QueueMonitoringBackendService = __decorate([
     __param(1, inject(SAAS_SERVICE_TYPES.AppConfigService)),
     __param(2, inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)),
     __metadata("design:paramtypes", [LoggerService,
-        AppConfigurationService,
-        RabbitMQAdapterBackendService])
+        AppConfigurationService, Object])
 ], QueueMonitoringBackendService);
 export { QueueMonitoringBackendService };
 //# sourceMappingURL=queue-monitoring.backend.service.js.map

package/dist/esm/queue/queue-monitoring.backend.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"queue-monitoring.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/queue-monitoring.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAChJ,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAGL,iBAAiB,GAElB,MAAM,uBAAuB,CAAC;AAYxB,IAAM,6BAA6B,GAAnC,MAAM,6BAA6B;IASrB;IAEA;IAEA;IAZT,QAAQ,CAA+D;IACzE,cAAc,GAAyB,EAAE,CAAC;IAC1C,eAAe,GAAqC,IAAI,GAAG,EAAE,CAAC;IAC9D,kBAAkB,GAA0B,IAAI,CAAC;IACjD,YAAY,GAAG,KAAK,CAAC;IAE7B,YAEmB,MAAqB,EAErB,aAAsC,EAEtC,eAA8C;QAJ9C,WAAM,GAAN,MAAM,CAAe;QAErB,kBAAa,GAAb,aAAa,CAAyB;QAEtC,oBAAe,GAAf,eAAe,CAA+B;QAE/D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,aAAqB,KAAK;QACxC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE3D,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE;oBACvD,GAAG,2BAA2B,CAAC,KAAK,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;QAEf,gBAAgB;QAChB,IAAI,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE;gBAC/D,GAAG,2BAA2B,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAA4B;QAClC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,SAAiB,EAAE,SAA8B;QACjE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,SAAiB;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAElD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC;YACH,+BAA+B;YAC/B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAEtD,2FAA2F;YAC3F,4FAA4F;YAC5F,sFAAsF;YACtF,oCAAoC;YAEpC,OAAO;gBACL,SAAS;gBACT,YAAY,EAAE,SAAS,CAAC,YAAY;gBACpC,eAAe,EAAE,CAAC,EAAE,+BAA+B;gBACnD,eAAe,EAAE,CAAC,EAAE,2BAA2B;gBAC/C,oBAAoB,EAAE,CAAC,EAAE,iCAAiC;gBAC1D,oBAAoB,EAAE,CAAC;gBACvB,mBAAmB,EAAE,CAAC;gBACtB,mBAAmB,EAAE,CAAC;gBACtB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,6BAA6B,EAAE;gBAC3C,SAAS;gBACT,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC;aAClC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC;QAC/C,uEAAuE;QACvE,MAAM,MAAM,GAAG;YACb,eAAe,KAAK,aAAa;YACjC,eAAe,KAAK,sBAAsB;YAC1C,eAAe,KAAK,YAAY;YAChC,eAAe,KAAK,YAAY;SACjC,CAAC;QAEF,IAAI,aAAa,GAAsB,iBAAiB,CAAC,OAAO,CAAC;QACjE,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,MAAM,YAAY,GAMb,EAAE,CAAC;QAER,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAEtD,mBAAmB;YACnB,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBAE5D,IAAI,MAAM,KAAK,iBAAiB,CAAC,SAAS,EAAE,CAAC;oBAC3C,aAAa,GAAG,iBAAiB,CAAC,SAAS,CAAC;oBAC5C,eAAe,EAAE,CAAC;oBAClB,YAAY,CAAC,IAAI,CAAC;wBAChB,SAAS;wBACT,SAAS,EAAE,wBAAwB;wBACnC,KAAK,EAAE,OAAO,CAAC,YAAY;wBAC3B,SAAS,EAAE,SAAS,CAAC,qBAAqB,IAAI,CAAC;wBAC/C,WAAW,EAAE,IAAI,IAAI,EAAE;qBACxB,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,MAAM,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;oBACjD,IAAI,aAAa,KAAK,iBAAiB,CAAC,OAAO,EAAE,CAAC;wBAChD,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC;oBAC7C,CAAC;oBACD,cAAc,EAAE,CAAC;oBACjB,YAAY,CAAC,IAAI,CAAC;wBAChB,SAAS;wBACT,SAAS,EAAE,gBAAgB;wBAC3B,KAAK,EAAE,OAAO,CAAC,YAAY;wBAC3B,SAAS,EAAE,SAAS,CAAC,qBAAqB,IAAI,CAAC;wBAC/C,WAAW,EAAE,IAAI,IAAI,EAAE;qBACxB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,aAAa;YACrB,aAAa;YACb,cAAc;YACd,eAAe;YACf,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,OAAqB,EACrB,SAA8B;QAE9B,gCAAgC;QAChC,IAAI,SAAS,CAAC,qBAAqB,KAAK,SAAS,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBAChE,OAAO,iBAAiB,CAAC,SAAS,CAAC;YACrC,CAAC;YACD,IAAI,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,qBAAqB,EAAE,CAAC;gBAC5D,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,SAAS,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAChD,IAAI,OAAO,CAAC,eAAe,IAAI,SAAS,CAAC,mBAAmB,EAAE,CAAC;gBAC7D,OAAO,iBAAiB,CAAC,SAAS,CAAC;YACrC,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,SAAS,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;gBACtD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,IAAI,SAAS,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,mBAAmB,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;gBAChE,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9C,IAAI,OAAO,CAAC,MAAM,KAAK,iBAAiB,CAAC,OAAO,EAAE,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,YAAY,CACrB,OAAO,CAAC,MAAM,KAAK,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,EACvE,KAAK,CAAC,SAAS,EACf;oBACE,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO;iBACR,CACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,SAAiB,EACjB,SAAiB,EACjB,OAAgC;QAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAE7E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE;oBAC3C,GAAG,2BAA2B,CAAC,KAAK,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB;QAC1C,OAAO;YACL,SAAS;YACT,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;YACvB,oBAAoB,EAAE,CAAC;YACvB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,IAAI,IAAI,EAAE;SACxB,CAAC;IACJ,CAAC;CACF,CAAA;AA/SY,6BAA6B;IADzC,UAAU,EAAE;IASR,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;qCAHzB,aAAa;QAEN,uBAAuB;QAErB,6BAA6B;GAbtD,6BAA6B,CA+SzC","sourcesContent":["/**\n * Queue Monitoring Backend Service\n *\n * Provides metrics, health checks, and alerting for the queue system.\n * Integrates with RabbitMQ for queue statistics and the job tracking system.\n */\n\nimport { injectable, inject } from 'inversify';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { describeCaughtError, projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';\nimport {\n  QueueMetrics,\n  QueueHealthSummary,\n  QueueHealthStatus,\n  QueueAlertThreshold,\n} from '@wildo-ai/saas-models';\n\n/**\n * Alert callback type for queue monitoring.\n */\nexport type QueueAlertCallback = (\n  alertType: string,\n  queueName: string,\n  details: Record<string, unknown>\n) => void | Promise<void>;\n\n@injectable()\nexport class QueueMonitoringBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private alertCallbacks: QueueAlertCallback[] = [];\n  private alertThresholds: Map<string, QueueAlertThreshold> = new Map();\n  private monitoringInterval: NodeJS.Timeout | null = null;\n  private isMonitoring = false;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.AppConfigService)\n    private readonly configService: AppConfigurationService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly rabbitMQAdapter: RabbitMQAdapterBackendService,\n  ) {\n    this.logDebug = this.logger.debugByType('QueueMonitoringBackendService');\n    this.logger.setDebugTypeLogging('QueueMonitoringBackendService', false);\n  }\n\n  /**\n   * Start monitoring queues at the specified interval.\n   */\n  startMonitoring(intervalMs: number = 30000): void {\n    if (this.isMonitoring) {\n      this.logger.warn('Queue monitoring is already running');\n      return;\n    }\n\n    this.isMonitoring = true;\n    this.logDebug('Starting queue monitoring', { intervalMs });\n\n    this.monitoringInterval = setInterval(async () => {\n      try {\n        await this.checkAllQueues();\n      } catch (error) {\n        this.logger.error('Error during queue monitoring check', {\n          ...projectCaughtErrorLogFields(error),\n        });\n      }\n    }, intervalMs);\n\n    // Initial check\n    this.checkAllQueues().catch((error) => {\n      this.logger.error('Error during initial queue monitoring check', {\n        ...projectCaughtErrorLogFields(error),\n      });\n    });\n  }\n\n  /**\n   * Stop monitoring queues.\n   */\n  stopMonitoring(): void {\n    if (this.monitoringInterval) {\n      clearInterval(this.monitoringInterval);\n      this.monitoringInterval = null;\n    }\n    this.isMonitoring = false;\n    this.logDebug('Queue monitoring stopped');\n  }\n\n  /**\n   * Register an alert callback.\n   */\n  onAlert(callback: QueueAlertCallback): void {\n    this.alertCallbacks.push(callback);\n  }\n\n  /**\n   * Set alert thresholds for a queue.\n   */\n  setAlertThreshold(queueName: string, threshold: QueueAlertThreshold): void {\n    this.alertThresholds.set(queueName, threshold);\n    this.logDebug('Alert threshold set', { queueName, threshold });\n  }\n\n  /**\n   * Get metrics for a specific queue.\n   * Also checks the corresponding dead letter queue if it exists.\n   */\n  async getQueueMetrics(queueName: string): Promise<QueueMetrics> {\n    const channel = this.rabbitMQAdapter.getChannel();\n\n    if (!channel) {\n      return this.createEmptyMetrics(queueName);\n    }\n\n    try {\n      // Get queue info from RabbitMQ\n      const queueInfo = await channel.checkQueue(queueName);\n\n      // Note: Dead letter queue check is skipped because channel.checkQueue() closes the channel\n      // if the queue doesn't exist. Only specific DLQ queues exist (operations.deadletter, etc.),\n      // not per-variant (operations.priority.deadletter). DLQ monitoring should be done via\n      // explicit queue name list instead.\n\n      return {\n        queueName,\n        pendingCount: queueInfo.messageCount,\n        processingCount: 0, // Would need consumer tracking\n        deadLetterCount: 0, // Skipped - see note above\n        publishRatePerMinute: 0, // Would need historical tracking\n        consumeRatePerMinute: 0,\n        avgProcessingTimeMs: 0,\n        p95ProcessingTimeMs: 0,\n        p99ProcessingTimeMs: 0,\n        errorRate: 0,\n        retryRate: 0,\n        collectedAt: new Date(),\n      };\n    } catch (error) {\n      // Queue might not exist yet\n      this.logDebug('Could not get queue metrics', {\n        queueName,\n        error: describeCaughtError(error),\n      });\n      return this.createEmptyMetrics(queueName);\n    }\n  }\n\n  /**\n   * Get health summary for all monitored queues.\n   */\n  async getHealthSummary(): Promise<QueueHealthSummary> {\n    const appId = this.configService.applicationId;\n    // Mono-env: queue names are wildo.queue.{appId}.{category}[.{variant}]\n    const queues = [\n      `wildo.queue.${appId}.operations`,\n      `wildo.queue.${appId}.operations.priority`,\n      `wildo.queue.${appId}.scheduled`,\n      `wildo.queue.${appId}.callbacks`,\n    ];\n\n    let overallStatus: QueueHealthStatus = QueueHealthStatus.HEALTHY;\n    let healthyQueues = 0;\n    let degradedQueues = 0;\n    let unhealthyQueues = 0;\n    const activeAlerts: Array<{\n      queueName: string;\n      alertType: string;\n      value: number;\n      threshold: number;\n      triggeredAt: Date;\n    }> = [];\n\n    for (const queueName of queues) {\n      const metrics = await this.getQueueMetrics(queueName);\n\n      // Check thresholds\n      const threshold = this.alertThresholds.get(queueName);\n      if (threshold) {\n        const status = this.evaluateQueueHealth(metrics, threshold);\n\n        if (status === QueueHealthStatus.UNHEALTHY) {\n          overallStatus = QueueHealthStatus.UNHEALTHY;\n          unhealthyQueues++;\n          activeAlerts.push({\n            queueName,\n            alertType: 'pending_count_exceeded',\n            value: metrics.pendingCount,\n            threshold: threshold.pendingCountThreshold ?? 0,\n            triggeredAt: new Date(),\n          });\n        } else if (status === QueueHealthStatus.DEGRADED) {\n          if (overallStatus === QueueHealthStatus.HEALTHY) {\n            overallStatus = QueueHealthStatus.DEGRADED;\n          }\n          degradedQueues++;\n          activeAlerts.push({\n            queueName,\n            alertType: 'queue_degraded',\n            value: metrics.pendingCount,\n            threshold: threshold.pendingCountThreshold ?? 0,\n            triggeredAt: new Date(),\n          });\n        } else {\n          healthyQueues++;\n        }\n      } else {\n        healthyQueues++;\n      }\n    }\n\n    return {\n      status: overallStatus,\n      healthyQueues,\n      degradedQueues,\n      unhealthyQueues,\n      activeAlerts,\n      checkedAt: new Date(),\n    };\n  }\n\n  /**\n   * Check if RabbitMQ connection is healthy.\n   */\n  async isRabbitMQHealthy(): Promise<boolean> {\n    return this.rabbitMQAdapter.isConnected();\n  }\n\n  /**\n   * Evaluate queue health based on thresholds.\n   */\n  private evaluateQueueHealth(\n    metrics: QueueMetrics,\n    threshold: QueueAlertThreshold\n  ): QueueHealthStatus {\n    // Check pending count threshold\n    if (threshold.pendingCountThreshold !== undefined) {\n      if (metrics.pendingCount >= threshold.pendingCountThreshold * 2) {\n        return QueueHealthStatus.UNHEALTHY;\n      }\n      if (metrics.pendingCount >= threshold.pendingCountThreshold) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    // Check dead letter threshold\n    if (threshold.deadLetterThreshold !== undefined) {\n      if (metrics.deadLetterCount >= threshold.deadLetterThreshold) {\n        return QueueHealthStatus.UNHEALTHY;\n      }\n    }\n\n    // Check error rate threshold\n    if (threshold.errorRateThreshold !== undefined) {\n      if (metrics.errorRate >= threshold.errorRateThreshold) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    // Check latency threshold\n    if (threshold.latencyThresholdMs !== undefined) {\n      if (metrics.p95ProcessingTimeMs >= threshold.latencyThresholdMs) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    return QueueHealthStatus.HEALTHY;\n  }\n\n  /**\n   * Check all queues and trigger alerts if needed.\n   */\n  private async checkAllQueues(): Promise<void> {\n    const summary = await this.getHealthSummary();\n\n    if (summary.status !== QueueHealthStatus.HEALTHY) {\n      for (const alert of summary.activeAlerts) {\n        await this.triggerAlert(\n          summary.status === QueueHealthStatus.UNHEALTHY ? 'critical' : 'warning',\n          alert.queueName,\n          {\n            alertType: alert.alertType,\n            value: alert.value,\n            threshold: alert.threshold,\n            summary,\n          }\n        );\n      }\n    }\n  }\n\n  /**\n   * Trigger an alert to all registered callbacks.\n   */\n  private async triggerAlert(\n    alertType: string,\n    queueName: string,\n    details: Record<string, unknown>\n  ): Promise<void> {\n    this.logger.warn('Queue alert triggered', { alertType, queueName, details });\n\n    for (const callback of this.alertCallbacks) {\n      try {\n        await callback(alertType, queueName, details);\n      } catch (error) {\n        this.logger.error('Error in alert callback', {\n          ...projectCaughtErrorLogFields(error),\n        });\n      }\n    }\n  }\n\n  /**\n   * Create empty metrics for a queue.\n   */\n  private createEmptyMetrics(queueName: string): QueueMetrics {\n    return {\n      queueName,\n      pendingCount: 0,\n      processingCount: 0,\n      deadLetterCount: 0,\n      publishRatePerMinute: 0,\n      consumeRatePerMinute: 0,\n      avgProcessingTimeMs: 0,\n      p95ProcessingTimeMs: 0,\n      p99ProcessingTimeMs: 0,\n      errorRate: 0,\n      retryRate: 0,\n      collectedAt: new Date(),\n    };\n  }\n}\n"]}
+{"version":3,"file":"queue-monitoring.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/queue-monitoring.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAChJ,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAE/G,OAAO,EAGL,iBAAiB,GAElB,MAAM,uBAAuB,CAAC;AAYxB,IAAM,6BAA6B,GAAnC,MAAM,6BAA6B;IASrB;IAEA;IAEA;IAZT,QAAQ,CAA+D;IACzE,cAAc,GAAyB,EAAE,CAAC;IAC1C,eAAe,GAAqC,IAAI,GAAG,EAAE,CAAC;IAC9D,kBAAkB,GAA0B,IAAI,CAAC;IACjD,YAAY,GAAG,KAAK,CAAC;IAE7B,YAEmB,MAAqB,EAErB,aAAsC,EAEtC,YAA0B;QAJ1B,WAAM,GAAN,MAAM,CAAe;QAErB,kBAAa,GAAb,aAAa,CAAyB;QAEtC,iBAAY,GAAZ,YAAY,CAAc;QAE3C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,+BAA+B,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,aAAqB,KAAK;QACxC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE3D,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE;oBACvD,GAAG,2BAA2B,CAAC,KAAK,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;QAEf,gBAAgB;QAChB,IAAI,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE;gBAC/D,GAAG,2BAA2B,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAA4B;QAClC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,SAAiB,EAAE,SAA8B;QACjE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,SAAiB;QACrC;;;;;;;;;;;WAWG;QACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAEtE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC;YACH,OAAO;gBACL,SAAS;gBACT,YAAY;gBACZ,eAAe,EAAE,CAAC,EAAE,+BAA+B;gBACnD,eAAe,EAAE,CAAC,EAAE,2BAA2B;gBAC/C,oBAAoB,EAAE,CAAC,EAAE,iCAAiC;gBAC1D,oBAAoB,EAAE,CAAC;gBACvB,mBAAmB,EAAE,CAAC;gBACtB,mBAAmB,EAAE,CAAC;gBACtB,mBAAmB,EAAE,CAAC;gBACtB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,6BAA6B,EAAE;gBAC3C,SAAS;gBACT,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC;aAClC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC;QAC/C,uEAAuE;QACvE,MAAM,MAAM,GAAG;YACb,eAAe,KAAK,aAAa;YACjC,eAAe,KAAK,sBAAsB;YAC1C,eAAe,KAAK,YAAY;YAChC,eAAe,KAAK,YAAY;SACjC,CAAC;QAEF,IAAI,aAAa,GAAsB,iBAAiB,CAAC,OAAO,CAAC;QACjE,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,MAAM,YAAY,GAMb,EAAE,CAAC;QAER,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAEtD,mBAAmB;YACnB,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBAE5D,IAAI,MAAM,KAAK,iBAAiB,CAAC,SAAS,EAAE,CAAC;oBAC3C,aAAa,GAAG,iBAAiB,CAAC,SAAS,CAAC;oBAC5C,eAAe,EAAE,CAAC;oBAClB,YAAY,CAAC,IAAI,CAAC;wBAChB,SAAS;wBACT,SAAS,EAAE,wBAAwB;wBACnC,KAAK,EAAE,OAAO,CAAC,YAAY;wBAC3B,SAAS,EAAE,SAAS,CAAC,qBAAqB,IAAI,CAAC;wBAC/C,WAAW,EAAE,IAAI,IAAI,EAAE;qBACxB,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,MAAM,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;oBACjD,IAAI,aAAa,KAAK,iBAAiB,CAAC,OAAO,EAAE,CAAC;wBAChD,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC;oBAC7C,CAAC;oBACD,cAAc,EAAE,CAAC;oBACjB,YAAY,CAAC,IAAI,CAAC;wBAChB,SAAS;wBACT,SAAS,EAAE,gBAAgB;wBAC3B,KAAK,EAAE,OAAO,CAAC,YAAY;wBAC3B,SAAS,EAAE,SAAS,CAAC,qBAAqB,IAAI,CAAC;wBAC/C,WAAW,EAAE,IAAI,IAAI,EAAE;qBACxB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,aAAa;YACrB,aAAa;YACb,cAAc;YACd,eAAe;YACf,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,OAAqB,EACrB,SAA8B;QAE9B,gCAAgC;QAChC,IAAI,SAAS,CAAC,qBAAqB,KAAK,SAAS,EAAE,CAAC;YAClD,IAAI,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBAChE,OAAO,iBAAiB,CAAC,SAAS,CAAC;YACrC,CAAC;YACD,IAAI,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,qBAAqB,EAAE,CAAC;gBAC5D,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,SAAS,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;YAChD,IAAI,OAAO,CAAC,eAAe,IAAI,SAAS,CAAC,mBAAmB,EAAE,CAAC;gBAC7D,OAAO,iBAAiB,CAAC,SAAS,CAAC;YACrC,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,SAAS,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;gBACtD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,IAAI,SAAS,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,mBAAmB,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;gBAChE,OAAO,iBAAiB,CAAC,QAAQ,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,iBAAiB,CAAC,OAAO,CAAC;IACnC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE9C,IAAI,OAAO,CAAC,MAAM,KAAK,iBAAiB,CAAC,OAAO,EAAE,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,YAAY,CACrB,OAAO,CAAC,MAAM,KAAK,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,EACvE,KAAK,CAAC,SAAS,EACf;oBACE,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,OAAO;iBACR,CACF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,SAAiB,EACjB,SAAiB,EACjB,OAAgC;QAEhC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAE7E,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE;oBAC3C,GAAG,2BAA2B,CAAC,KAAK,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB;QAC1C,OAAO;YACL,SAAS;YACT,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;YACvB,oBAAoB,EAAE,CAAC;YACvB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,IAAI,IAAI,EAAE;SACxB,CAAC;IACJ,CAAC;CACF,CAAA;AAvTY,6BAA6B;IADzC,UAAU,EAAE;IASR,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;qCAHzB,aAAa;QAEN,uBAAuB;GAX9C,6BAA6B,CAuTzC","sourcesContent":["/**\n * Queue Monitoring Backend Service\n *\n * Provides metrics, health checks, and alerting for the queue system.\n * Integrates with RabbitMQ for queue statistics and the job tracking system.\n */\n\nimport { injectable, inject } from 'inversify';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { describeCaughtError, projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport type { QueueAdapter } from './adapters/queue-adapter.backend.interface';\nimport {\n  QueueMetrics,\n  QueueHealthSummary,\n  QueueHealthStatus,\n  QueueAlertThreshold,\n} from '@wildo-ai/saas-models';\n\n/**\n * Alert callback type for queue monitoring.\n */\nexport type QueueAlertCallback = (\n  alertType: string,\n  queueName: string,\n  details: Record<string, unknown>\n) => void | Promise<void>;\n\n@injectable()\nexport class QueueMonitoringBackendService {\n  protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n  private alertCallbacks: QueueAlertCallback[] = [];\n  private alertThresholds: Map<string, QueueAlertThreshold> = new Map();\n  private monitoringInterval: NodeJS.Timeout | null = null;\n  private isMonitoring = false;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.Logger)\n    private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.AppConfigService)\n    private readonly configService: AppConfigurationService,\n    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n    private readonly queueAdapter: QueueAdapter,\n  ) {\n    this.logDebug = this.logger.debugByType('QueueMonitoringBackendService');\n    this.logger.setDebugTypeLogging('QueueMonitoringBackendService', false);\n  }\n\n  /**\n   * Start monitoring queues at the specified interval.\n   */\n  startMonitoring(intervalMs: number = 30000): void {\n    if (this.isMonitoring) {\n      this.logger.warn('Queue monitoring is already running');\n      return;\n    }\n\n    this.isMonitoring = true;\n    this.logDebug('Starting queue monitoring', { intervalMs });\n\n    this.monitoringInterval = setInterval(async () => {\n      try {\n        await this.checkAllQueues();\n      } catch (error) {\n        this.logger.error('Error during queue monitoring check', {\n          ...projectCaughtErrorLogFields(error),\n        });\n      }\n    }, intervalMs);\n\n    // Initial check\n    this.checkAllQueues().catch((error) => {\n      this.logger.error('Error during initial queue monitoring check', {\n        ...projectCaughtErrorLogFields(error),\n      });\n    });\n  }\n\n  /**\n   * Stop monitoring queues.\n   */\n  stopMonitoring(): void {\n    if (this.monitoringInterval) {\n      clearInterval(this.monitoringInterval);\n      this.monitoringInterval = null;\n    }\n    this.isMonitoring = false;\n    this.logDebug('Queue monitoring stopped');\n  }\n\n  /**\n   * Register an alert callback.\n   */\n  onAlert(callback: QueueAlertCallback): void {\n    this.alertCallbacks.push(callback);\n  }\n\n  /**\n   * Set alert thresholds for a queue.\n   */\n  setAlertThreshold(queueName: string, threshold: QueueAlertThreshold): void {\n    this.alertThresholds.set(queueName, threshold);\n    this.logDebug('Alert threshold set', { queueName, threshold });\n  }\n\n  /**\n   * Get metrics for a specific queue.\n   * Also checks the corresponding dead letter queue if it exists.\n   */\n  async getQueueMetrics(queueName: string): Promise<QueueMetrics> {\n    /*\n     * Asks the ADAPTER how deep the queue is, rather than reaching through it for an amqplib\n     * channel (#502). Behaviour is unchanged for RabbitMQ — `null` here is what a null channel\n     * produced before, and it still yields empty metrics — but the question is now one every\n     * adapter can answer about itself, so a broker-free runtime reports real numbers instead of\n     * being structurally unable to be monitored.\n     *\n     * Dead-letter depth stays unreported: on RabbitMQ `checkQueue` CLOSES the channel when the\n     * queue does not exist, and only specific DLQs exist (`operations.deadletter`, not the\n     * per-variant `operations.priority.deadletter`). That needs an explicit queue-name list, and it\n     * is unrelated to this change.\n     */\n    const pendingCount = await this.queueAdapter.getQueueDepth(queueName);\n\n    if (pendingCount === null) {\n      return this.createEmptyMetrics(queueName);\n    }\n\n    try {\n      return {\n        queueName,\n        pendingCount,\n        processingCount: 0, // Would need consumer tracking\n        deadLetterCount: 0, // Skipped - see note above\n        publishRatePerMinute: 0, // Would need historical tracking\n        consumeRatePerMinute: 0,\n        avgProcessingTimeMs: 0,\n        p95ProcessingTimeMs: 0,\n        p99ProcessingTimeMs: 0,\n        errorRate: 0,\n        retryRate: 0,\n        collectedAt: new Date(),\n      };\n    } catch (error) {\n      // Queue might not exist yet\n      this.logDebug('Could not get queue metrics', {\n        queueName,\n        error: describeCaughtError(error),\n      });\n      return this.createEmptyMetrics(queueName);\n    }\n  }\n\n  /**\n   * Get health summary for all monitored queues.\n   */\n  async getHealthSummary(): Promise<QueueHealthSummary> {\n    const appId = this.configService.applicationId;\n    // Mono-env: queue names are wildo.queue.{appId}.{category}[.{variant}]\n    const queues = [\n      `wildo.queue.${appId}.operations`,\n      `wildo.queue.${appId}.operations.priority`,\n      `wildo.queue.${appId}.scheduled`,\n      `wildo.queue.${appId}.callbacks`,\n    ];\n\n    let overallStatus: QueueHealthStatus = QueueHealthStatus.HEALTHY;\n    let healthyQueues = 0;\n    let degradedQueues = 0;\n    let unhealthyQueues = 0;\n    const activeAlerts: Array<{\n      queueName: string;\n      alertType: string;\n      value: number;\n      threshold: number;\n      triggeredAt: Date;\n    }> = [];\n\n    for (const queueName of queues) {\n      const metrics = await this.getQueueMetrics(queueName);\n\n      // Check thresholds\n      const threshold = this.alertThresholds.get(queueName);\n      if (threshold) {\n        const status = this.evaluateQueueHealth(metrics, threshold);\n\n        if (status === QueueHealthStatus.UNHEALTHY) {\n          overallStatus = QueueHealthStatus.UNHEALTHY;\n          unhealthyQueues++;\n          activeAlerts.push({\n            queueName,\n            alertType: 'pending_count_exceeded',\n            value: metrics.pendingCount,\n            threshold: threshold.pendingCountThreshold ?? 0,\n            triggeredAt: new Date(),\n          });\n        } else if (status === QueueHealthStatus.DEGRADED) {\n          if (overallStatus === QueueHealthStatus.HEALTHY) {\n            overallStatus = QueueHealthStatus.DEGRADED;\n          }\n          degradedQueues++;\n          activeAlerts.push({\n            queueName,\n            alertType: 'queue_degraded',\n            value: metrics.pendingCount,\n            threshold: threshold.pendingCountThreshold ?? 0,\n            triggeredAt: new Date(),\n          });\n        } else {\n          healthyQueues++;\n        }\n      } else {\n        healthyQueues++;\n      }\n    }\n\n    return {\n      status: overallStatus,\n      healthyQueues,\n      degradedQueues,\n      unhealthyQueues,\n      activeAlerts,\n      checkedAt: new Date(),\n    };\n  }\n\n  /**\n   * Whether the queue substrate this runtime uses is connected.\n   *\n   * Named `isRabbitMQHealthy` for its callers' sake and answered by whichever adapter is bound\n   * (#502) — an in-memory queue reports connected once `connect()` has run, which is the honest\n   * answer to \"can this runtime enqueue work\" regardless of what is behind it.\n   */\n  async isRabbitMQHealthy(): Promise<boolean> {\n    return this.queueAdapter.isConnected();\n  }\n\n  /**\n   * Evaluate queue health based on thresholds.\n   */\n  private evaluateQueueHealth(\n    metrics: QueueMetrics,\n    threshold: QueueAlertThreshold\n  ): QueueHealthStatus {\n    // Check pending count threshold\n    if (threshold.pendingCountThreshold !== undefined) {\n      if (metrics.pendingCount >= threshold.pendingCountThreshold * 2) {\n        return QueueHealthStatus.UNHEALTHY;\n      }\n      if (metrics.pendingCount >= threshold.pendingCountThreshold) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    // Check dead letter threshold\n    if (threshold.deadLetterThreshold !== undefined) {\n      if (metrics.deadLetterCount >= threshold.deadLetterThreshold) {\n        return QueueHealthStatus.UNHEALTHY;\n      }\n    }\n\n    // Check error rate threshold\n    if (threshold.errorRateThreshold !== undefined) {\n      if (metrics.errorRate >= threshold.errorRateThreshold) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    // Check latency threshold\n    if (threshold.latencyThresholdMs !== undefined) {\n      if (metrics.p95ProcessingTimeMs >= threshold.latencyThresholdMs) {\n        return QueueHealthStatus.DEGRADED;\n      }\n    }\n\n    return QueueHealthStatus.HEALTHY;\n  }\n\n  /**\n   * Check all queues and trigger alerts if needed.\n   */\n  private async checkAllQueues(): Promise<void> {\n    const summary = await this.getHealthSummary();\n\n    if (summary.status !== QueueHealthStatus.HEALTHY) {\n      for (const alert of summary.activeAlerts) {\n        await this.triggerAlert(\n          summary.status === QueueHealthStatus.UNHEALTHY ? 'critical' : 'warning',\n          alert.queueName,\n          {\n            alertType: alert.alertType,\n            value: alert.value,\n            threshold: alert.threshold,\n            summary,\n          }\n        );\n      }\n    }\n  }\n\n  /**\n   * Trigger an alert to all registered callbacks.\n   */\n  private async triggerAlert(\n    alertType: string,\n    queueName: string,\n    details: Record<string, unknown>\n  ): Promise<void> {\n    this.logger.warn('Queue alert triggered', { alertType, queueName, details });\n\n    for (const callback of this.alertCallbacks) {\n      try {\n        await callback(alertType, queueName, details);\n      } catch (error) {\n        this.logger.error('Error in alert callback', {\n          ...projectCaughtErrorLogFields(error),\n        });\n      }\n    }\n  }\n\n  /**\n   * Create empty metrics for a queue.\n   */\n  private createEmptyMetrics(queueName: string): QueueMetrics {\n    return {\n      queueName,\n      pendingCount: 0,\n      processingCount: 0,\n      deadLetterCount: 0,\n      publishRatePerMinute: 0,\n      consumeRatePerMinute: 0,\n      avgProcessingTimeMs: 0,\n      p95ProcessingTimeMs: 0,\n      p99ProcessingTimeMs: 0,\n      errorRate: 0,\n      retryRate: 0,\n      collectedAt: new Date(),\n    };\n  }\n}\n"]}

package/dist/esm/queue/queue-worker-initializer.backend.service.d.ts

@@ -1,8 +1,21 @@
 /**
+ * @wildo_source:part:start saas.backend.queue.worker-topology facet:layer:backend facet:audience:app-developer
+ *
+ * Where an application's background jobs actually run.
+ *
+ * ⚠️ **Queue consumers are NOT a separate tier.** They run inside each backend API pod, so every pod
+ * both ENQUEUES jobs (as an API handler) and CONSUMES them (as a worker). The consequence is the one
+ * that matters when a queue backs up: **scaling job processing means scaling the BACKEND**, not
+ * adding worker processes.
+ *
+ * ⚠️ **A declared `workers:` entry is a different thing entirely, and its name misleads.** That is a
+ * standalone HTTP service the backend CALLS — it never connects to the broker, has no database and
+ * no cache, and cannot read or write application resources except through what the caller puts in the
+ * request. Adding one does not add queue capacity.
+ *
  * Queue Worker Initializer Backend Service
  *
  * Initializes queue workers within the backend API application.
- * Workers are NOT separate microservices - they run inside each backend API pod.
  *
  * This service is called during application startup to:
  * 1. Connect to RabbitMQ
@@ -13,6 +26,8 @@
  * Every backend API pod can both:
  * - Enqueue jobs (as API handlers)
  * - Consume and execute jobs (as workers)
+ *
+ * @wildo_source:part:end saas.backend.queue.worker-topology
  */
 import { LoggerService } from '../monitoring/logger/services/logger.service';
 import { AppConfigurationService } from '../config/services/app-configuration.backend.service';

package/dist/esm/queue/queue-worker-initializer.backend.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"queue-worker-initializer.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/queue-worker-initializer.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAE7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;;;;;;;;;;;;;;;OAkBG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,2DAA2D;IAC3D,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AASD,qBACa,oCAAoC;IAO7C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAEjC,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAE5B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAEhC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAhB7B,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAA+C;gBAI1C,MAAM,EAAE,aAAa,EAErB,gBAAgB,EAAE,uBAAuB,EAEzC,YAAY,EAAE,mBAAmB,EAEjC,WAAW,EAAE,yBAAyB,EAEtC,eAAe,EAAE,mCAAmC,EAEpD,UAAU,EAAE,6BAA6B;IAM5D;;;;;OAKG;IACG,UAAU,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkF3D;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAuBhC"}
+{"version":3,"file":"queue-worker-initializer.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/queue-worker-initializer.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAE7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,EAAE,mCAAmC,EAAE,MAAM,2CAA2C,CAAC;AAChG,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;;;;;;;;;;;;;;;OAkBG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,2DAA2D;IAC3D,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AASD,qBACa,oCAAoC;IAO7C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAEjC,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAE5B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAEhC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAhB7B,SAAS,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IACjF,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAA+C;gBAI1C,MAAM,EAAE,aAAa,EAErB,gBAAgB,EAAE,uBAAuB,EAEzC,YAAY,EAAE,mBAAmB,EAEjC,WAAW,EAAE,yBAAyB,EAEtC,eAAe,EAAE,mCAAmC,EAEpD,UAAU,EAAE,6BAA6B;IAM5D;;;;;OAKG;IACG,UAAU,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkF3D;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAuBhC"}