@geminixiang/mikan 1.0.0-beta.17 → 1.0.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"gondolin-fleet.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-fleet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAA+B,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EACL,wBAAwB,EACxB,8BAA8B,GAE/B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,wBAAwB,GAIzB,MAAM,6BAA6B,CAAC;AAGrC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,aAAa,GAAG,IAAI,CAAC;AAqC3B,SAAS,gBAAgB,CAAC,QAAgC;IACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAa,EAAE,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,MAAM;QACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,GAAG;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;QACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC9C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC3C,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC7D,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;KACxD,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,mBAAmB;IAAzB;QAEU,YAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzC,wBAAmB,GAA4B,EAAE,CAAC;QAClD,eAAU,GAA2B,kBAAkB,CAAC;QACxD,eAAU,GAAG,iBAAiB,GAAG,IAAI,CAAC;QACtC,gBAAW,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACxC,gBAAW,GAAG,aAAa,CAAC;QAC5B,QAAG,GAAiB,IAAI,CAAC,GAAG,CAAC;QAC7B,UAAK,GAAkC,CAAC,EAAE,EAAE,EAAE,CACpD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAwRtD,CAAC;IAtRC,SAAS,CAAC,QAAiC,EAAE,SAAkC;QAC7E,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,kBAAkB,CAAC;QAC9D,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,WAAW,GAAG,SAAS,EAAE,WAAW,IAAI,aAAa,CAAC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,SAAS,EAAE,mBAAmB,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,CAAC,SAAS,EAAE,mBAAmB,EAAE,eAAe,IAAI,iBAAiB,CAAC,GAAG,IAAI,CAAC;QAChG,IAAI,CAAC,WAAW,GAAG,CAAC,QAAQ,EAAE,gBAAgB,IAAI,kBAAkB,CAAC,GAAG,IAAI,CAAC;QAC7E,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC/C,OAAO,CAAC,0DAA0D;QACpE,CAAC;QACD,MAAM,gBAAgB,GACpB,SAAS,EAAE,gBAAgB;YAC3B,CAAC,CAAC,MAAoC,EAAE,mBAA4C,EAAE,EAAE,CACtF,IAAI,wBAAwB,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,MAAM,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACnC,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,EAAE;gBAC1C,GAAG,SAAS,EAAE,mBAAmB;gBACjC,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;oBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;wBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,YAAY,CACV,QAAsC,EACtC,gBAAkF;QAElF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,GAAG,IAAI,CAAC,mBAAmB;YAC3B,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;oBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,IAAY;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;IACtC,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrD,IAAI,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3E,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB,EAAE,IAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,GAAG,YAAY,8BAA8B,CAAC;wBAAE,MAAM,GAAG,CAAC;oBAChE,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,qBAAqB,CACxB,UAAU,EACV,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,cAAc,EACrB,IAAI,8BAA8B,CAAC,gCAAgC,CAAC,CACrE,CAAC;YACJ,CAAC;YACD,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,MAAM,uBAAuB,UAAU,iBAAiB,CACpF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA4F;QAE5F,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,uEAAuE;YACvE,6DAA6D;YAC7D,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,8BAA8B,EAAE,CAAC;gBAClD,qEAAqE;gBACrE,qEAAqE;gBACrE,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,IAAI,iCAAiC,MAAM,CAAC,UAAU,+BAA+B,CACjH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA6B,EAC7B,OAAe,EACf,OAAO,GAA2D,EAAE;QAEpE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,wBAAwB,CAAC,gCAAgC,CAAC,CAAC;QAClF,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAA6B;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,OAAO,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,QAA0D,CAAC;YAC/D,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,iDAAiD;YAC7D,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACvD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,GAAG,CAAC,OAAO,CACT,uCAAuC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,GAAG,CACxF,CAAC;oBACF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;oBACnF,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI;oBAAE,SAAS;gBAC5C,GAAG,CAAC,OAAO,CACT,oCAAoC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,IAAI,CACpH,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;wBAC3B,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;wBAC9B,SAAS,EAAE,CAAC;qBACb,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,UAAU,CACZ,iCAAiC,OAAO,CAAC,UAAU,SAAS,MAAM,CAAC,IAAI,GAAG,EAC1E,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,MAAmB,EACnB,UAAkB,EAClB,IAAyB;QAEzB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3E,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAEO,qBAAqB,CAC3B,UAAkB,EAClB,UAAkB,EAClB,cAAsB,EACtB,KAAY;QAEZ,MAAM,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,WAAW,IAAI,CAAC;YAAE,OAAO;QAC7B,yEAAyE;QACzE,qEAAqE;QACrE,wDAAwD;QACxD,MAAM,IAAI,YAAY,CACpB,2BAA2B,UAAU,qBAAqB,KAAK,CAAC,OAAO,KAAK;YAC1E,aAAa,UAAU,kBAAkB,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,8BAA8B,CACvG,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/C,SAAS,CAAC;YACR,IAAI,IAAuD,CAAC;YAC5D,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC7B,QAAQ,IAAI,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,MAAc,CAAC;gBACnB,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;oBAChD,MAAM,GAAG,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;gBACD,SAAS,IAAI,CAAC,CAAC;gBACf,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACxC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBACvC,IAAI,IAAI,CAAC,CAAC;oBACV,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBACjF,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI;oBAAE,IAAI,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACzD,CAAC;YACD,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC,MAAM,CAAC;YAC7B,IAAI,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACtC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,IAAI,YAAY,CAAC,gEAAgE,CAAC,CAAC;YAC3F,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,IAAI,YAAY,CACpB,8DAA8D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CACtG,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,UAA8B;QAC9C,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,mBAAmB,EAAE,CAAC","sourcesContent":["import * as log from \"../log.js\";\nimport type { GondolinRemoteSettings, GondolinRemoteWorkerSettings } from \"../types.js\";\nimport { SandboxError } from \"./errors.js\";\nimport { gondolinPlacements, type GondolinPlacementStore } from \"./gondolin-placement.js\";\nimport {\n GondolinRemoteConnection,\n GondolinWorkerUnreachableError,\n type GondolinRemoteOverrides,\n} from \"./gondolin-remote.js\";\nimport {\n GondolinRuntimeGoneError,\n type GondolinRuntimeHandle,\n type GondolinRuntimeSpec,\n type GondolinRuntimeTransport,\n} from \"./gondolin-worker-client.js\";\nimport type { ExecResult } from \"./types.js\";\n\nconst LEASE_TTL_SECONDS = 300;\nconst QUEUE_WAIT_SECONDS = 60;\nconst QUEUE_POLL_MS = 2000;\n\n/** The connection surface the fleet needs from each worker. */\nexport interface GondolinWorkerConnection {\n health(): Promise<Record<string, unknown>>;\n ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle>;\n stop(\n handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\">,\n ): Promise<void>;\n exec(\n handle: GondolinRuntimeHandle,\n command: string,\n options?: { env?: Record<string, string>; signal?: AbortSignal },\n ): Promise<ExecResult>;\n isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean>;\n listRuntimes(): Promise<Array<{ sessionId: string; instanceId: string }>>;\n dispose(): void;\n}\n\ninterface FleetWorker {\n name: string;\n settings: GondolinRemoteWorkerSettings;\n connection: GondolinWorkerConnection;\n}\n\ninterface GondolinFleetOverrides {\n createConnection?: (\n worker: GondolinRemoteWorkerSettings,\n connectionOverrides: GondolinRemoteOverrides,\n ) => GondolinWorkerConnection;\n placements?: GondolinPlacementStore;\n connectionOverrides?: GondolinRemoteOverrides;\n now?: () => number;\n queuePollMs?: number;\n sleep?: (ms: number) => Promise<void>;\n}\n\nfunction normalizeWorkers(settings: GondolinRemoteSettings): GondolinRemoteWorkerSettings[] {\n const inline = settings.workers ?? [{ url: settings.url as string }];\n return inline.map((worker) => ({\n ...worker,\n name: worker.name ?? worker.url,\n caFile: worker.caFile ?? settings.caFile,\n certFile: worker.certFile ?? settings.certFile,\n keyFile: worker.keyFile ?? settings.keyFile,\n workspaceRoot: worker.workspaceRoot ?? settings.workspaceRoot,\n maxRuntimes: worker.maxRuntimes ?? settings.maxRuntimes,\n }));\n}\n\n/**\n * Places each conversation's runtime on one worker of the fleet and keeps it\n * there (sticky placement). Failover to another worker happens only when the\n * placed worker is unreachable AND its lease fencing watermark has passed —\n * the point after which the old worker's janitor has provably stopped the\n * runtime, so shared storage never sees two writers.\n */\nclass GondolinFleetClient implements GondolinRuntimeTransport {\n private settings?: GondolinRemoteSettings;\n private workers = new Map<string, FleetWorker>();\n private connectionOverrides: GondolinRemoteOverrides = {};\n private placements: GondolinPlacementStore = gondolinPlacements;\n private leaseTtlMs = LEASE_TTL_SECONDS * 1000;\n private queueWaitMs = QUEUE_WAIT_SECONDS * 1000;\n private queuePollMs = QUEUE_POLL_MS;\n private now: () => number = Date.now;\n private sleep: (ms: number) => Promise<void> = (ms) =>\n new Promise((resolve) => setTimeout(resolve, ms));\n\n configure(settings?: GondolinRemoteSettings, overrides?: GondolinFleetOverrides): void {\n for (const worker of this.workers.values()) worker.connection.dispose();\n this.workers.clear();\n this.settings = settings;\n this.placements = overrides?.placements ?? gondolinPlacements;\n this.now = overrides?.now ?? Date.now;\n this.sleep = overrides?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));\n this.queuePollMs = overrides?.queuePollMs ?? QUEUE_POLL_MS;\n this.connectionOverrides = overrides?.connectionOverrides ?? {};\n this.leaseTtlMs = (overrides?.connectionOverrides?.leaseTtlSeconds ?? LEASE_TTL_SECONDS) * 1000;\n this.queueWaitMs = (settings?.queueWaitSeconds ?? QUEUE_WAIT_SECONDS) * 1000;\n if (!settings) return;\n if (!settings.workers?.length && !settings.url) {\n return; // not configured; validate() reports the actionable error\n }\n const createConnection =\n overrides?.createConnection ??\n ((worker: GondolinRemoteWorkerSettings, connectionOverrides: GondolinRemoteOverrides) =>\n new GondolinRemoteConnection(worker, connectionOverrides));\n for (const worker of normalizeWorkers(settings)) {\n const name = worker.name as string;\n const connection = createConnection(worker, {\n ...overrides?.connectionOverrides,\n onLeaseActivity: (instanceId, expiresAtMs) => {\n if (this.placements.get(instanceId)?.worker === name) {\n this.placements.touch(instanceId, expiresAtMs);\n }\n },\n });\n this.workers.set(name, { name, settings: worker, connection });\n }\n }\n\n isConfigured(): boolean {\n return this.workers.size > 0;\n }\n\n /**\n * Attach a dial-home worker at runtime (the gateway calls this when a\n * worker registers). The factory receives the fleet's connection overrides\n * plus the lease-watermark callback, mirroring configure()'s wiring.\n */\n attachWorker(\n settings: GondolinRemoteWorkerSettings,\n createConnection: (overrides: GondolinRemoteOverrides) => GondolinWorkerConnection,\n ): void {\n const name = settings.name ?? settings.url;\n this.workers.get(name)?.connection.dispose();\n const connection = createConnection({\n ...this.connectionOverrides,\n onLeaseActivity: (instanceId, expiresAtMs) => {\n if (this.placements.get(instanceId)?.worker === name) {\n this.placements.touch(instanceId, expiresAtMs);\n }\n },\n });\n this.workers.set(name, { name, settings: { ...settings, name }, connection });\n }\n\n /**\n * Detach a dial-home worker (disconnect). Placements survive: their lease\n * watermarks keep fencing failover until expiry, exactly as for an\n * unreachable static worker.\n */\n detachWorker(name: string): void {\n const worker = this.workers.get(name);\n if (!worker) return;\n worker.connection.dispose();\n this.workers.delete(name);\n }\n\n imageSelector(): string | undefined {\n return this.settings?.imageSelector;\n }\n\n /** At least one worker must answer for the sandbox mode to come up. */\n async health(): Promise<Array<{ worker: string; health?: Record<string, unknown> }>> {\n const report = await Promise.all(\n Array.from(this.workers.values()).map(async (worker) => {\n try {\n return { worker: worker.name, health: await worker.connection.health() };\n } catch {\n return { worker: worker.name };\n }\n }),\n );\n if (report.length > 0 && !report.some((entry) => entry.health)) {\n throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n }\n return report;\n }\n\n async ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle> {\n const placed = this.placements.get(instanceId);\n if (placed) {\n const worker = this.workers.get(placed.worker);\n if (worker) {\n try {\n return await this.ensureOn(worker, instanceId, spec);\n } catch (err) {\n if (!(err instanceof GondolinWorkerUnreachableError)) throw err;\n this.assertFailoverAllowed(instanceId, placed.worker, placed.leaseExpiresAt, err);\n }\n } else {\n this.assertFailoverAllowed(\n instanceId,\n placed.worker,\n placed.leaseExpiresAt,\n new GondolinWorkerUnreachableError(\"worker is no longer configured\"),\n );\n }\n log.logWarning(\n `Gondolin worker '${placed.worker}' lost runtime for '${instanceId}'; failing over`,\n );\n }\n const worker = await this.selectWorker();\n return this.ensureOn(worker, instanceId, spec);\n }\n\n async stop(\n handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\" | \"workerName\">,\n ): Promise<void> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) {\n // the worker left the fleet config; its janitor stops the runtime when\n // the lease expires, and the record keeps fencing until then\n return;\n }\n try {\n await worker.connection.stop(handle);\n this.placements.delete(handle.instanceId);\n } catch (err) {\n if (err instanceof GondolinWorkerUnreachableError) {\n // cannot confirm the stop; keep the placement so its watermark still\n // fences any failover, and let the daemon's expiry janitor finish it\n log.logWarning(\n `Gondolin worker '${worker.name}' unreachable while stopping '${handle.instanceId}'; leaving it to lease expiry`,\n );\n return;\n }\n throw err;\n }\n }\n\n async exec(\n handle: GondolinRuntimeHandle,\n command: string,\n options: { env?: Record<string, string>; signal?: AbortSignal } = {},\n ): Promise<ExecResult> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) throw new GondolinRuntimeGoneError(\"worker is no longer configured\");\n return worker.connection.exec(handle, command, options);\n }\n\n async isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) return false;\n return worker.connection.isRuntimeAlive(handle);\n }\n\n /**\n * Fleet reconciliation: a runtime whose conversation is placed on a\n * different worker is a stray from a superseded placement — stop it. A\n * runtime with no placement at all (crash between daemon ensure and the\n * placement write) is adopted instead of killed.\n */\n async reconcile(): Promise<void> {\n for (const worker of this.workers.values()) {\n let runtimes: Array<{ sessionId: string; instanceId: string }>;\n try {\n runtimes = await worker.connection.listRuntimes();\n } catch {\n continue; // unreachable workers reconcile on a later sweep\n }\n for (const runtime of runtimes) {\n const record = this.placements.get(runtime.instanceId);\n if (!record) {\n log.logInfo(\n `Adopting unplaced Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}'`,\n );\n this.placements.set(runtime.instanceId, worker.name, this.now() + this.leaseTtlMs);\n continue;\n }\n if (record.worker === worker.name) continue;\n log.logInfo(\n `Stopping stray Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}' (placed on '${record.worker}')`,\n );\n try {\n await worker.connection.stop({\n sessionId: runtime.sessionId,\n instanceId: runtime.instanceId,\n workerPid: 0,\n });\n } catch (err) {\n log.logWarning(\n `Failed to stop stray runtime '${runtime.instanceId}' on '${worker.name}'`,\n err instanceof Error ? err.message : String(err),\n );\n }\n }\n }\n }\n\n private async ensureOn(\n worker: FleetWorker,\n instanceId: string,\n spec: GondolinRuntimeSpec,\n ): Promise<GondolinRuntimeHandle> {\n const handle = await worker.connection.ensure(instanceId, spec);\n this.placements.set(instanceId, worker.name, this.now() + this.leaseTtlMs);\n return { ...handle, workerName: worker.name };\n }\n\n private assertFailoverAllowed(\n instanceId: string,\n workerName: string,\n leaseExpiresAt: number,\n cause: Error,\n ): void {\n const remainingMs = leaseExpiresAt - this.now();\n if (remainingMs <= 0) return;\n // Failing over earlier could put a second writable VM on the same shared\n // workspace: the old worker may be alive behind a partition, and its\n // daemon only stops the runtime once the lease expires.\n throw new SandboxError(\n `Error: Gondolin worker '${workerName}' is unreachable (${cause.message}); ` +\n `retrying '${instanceId}' elsewhere in ${Math.ceil(remainingMs / 1000)}s once its lease has expired`,\n );\n }\n\n private async selectWorker(): Promise<FleetWorker> {\n const deadline = this.now() + this.queueWaitMs;\n for (;;) {\n let best: { worker: FleetWorker; load: number } | undefined;\n let reachable = 0;\n let draining = 0;\n let full = 0;\n for (const worker of this.workers.values()) {\n if (worker.settings.draining) {\n draining += 1;\n continue;\n }\n let active: number;\n try {\n const health = await worker.connection.health();\n active = typeof health.activeRuntimes === \"number\" ? health.activeRuntimes : 0;\n } catch {\n continue;\n }\n reachable += 1;\n const cap = worker.settings.maxRuntimes;\n if (cap !== undefined && active >= cap) {\n full += 1;\n continue;\n }\n const load = cap !== undefined ? active / cap : active / Number.MAX_SAFE_INTEGER;\n if (!best || load < best.load) best = { worker, load };\n }\n if (best) return best.worker;\n if (reachable === 0 && draining === 0) {\n throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n }\n if (full === 0) {\n throw new SandboxError(\"Error: every gondolin remote worker is draining or unreachable\");\n }\n if (this.now() >= deadline) {\n throw new SandboxError(\n `Error: all gondolin remote workers are at capacity (waited ${Math.round(this.queueWaitMs / 1000)}s)`,\n );\n }\n await this.sleep(this.queuePollMs);\n }\n }\n\n private workerFor(workerName: string | undefined): FleetWorker | undefined {\n if (!workerName) return undefined;\n return this.workers.get(workerName);\n }\n}\n\nexport const gondolinFleet = new GondolinFleetClient();\n"]}
1
+ {"version":3,"file":"gondolin-fleet.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-fleet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAA+B,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EACL,wBAAwB,EACxB,8BAA8B,GAE/B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,wBAAwB,GAIzB,MAAM,6BAA6B,CAAC;AAGrC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,aAAa,GAAG,IAAI,CAAC;AAuC3B,SAAS,gBAAgB,CAAC,QAAgC;IACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAa,EAAE,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,MAAM;QACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,GAAG;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;QACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC9C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC3C,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC7D,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;KACxD,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,mBAAmB;IAAzB;QAEU,YAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzC,wBAAmB,GAA4B,EAAE,CAAC;QAClD,eAAU,GAA2B,kBAAkB,CAAC;QACxD,eAAU,GAAG,iBAAiB,GAAG,IAAI,CAAC;QACtC,gBAAW,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACxC,gBAAW,GAAG,aAAa,CAAC;QAC5B,QAAG,GAAiB,IAAI,CAAC,GAAG,CAAC;QAC7B,UAAK,GAAkC,CAAC,EAAE,EAAE,EAAE,CACpD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IA8UtD,CAAC;IA5UC,SAAS,CAAC,QAAiC,EAAE,SAAkC;QAC7E,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,kBAAkB,CAAC;QAC9D,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,WAAW,GAAG,SAAS,EAAE,WAAW,IAAI,aAAa,CAAC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,SAAS,EAAE,mBAAmB,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,CAAC,SAAS,EAAE,mBAAmB,EAAE,eAAe,IAAI,iBAAiB,CAAC,GAAG,IAAI,CAAC;QAChG,IAAI,CAAC,WAAW,GAAG,CAAC,QAAQ,EAAE,gBAAgB,IAAI,kBAAkB,CAAC,GAAG,IAAI,CAAC;QAC7E,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC/C,OAAO,CAAC,0DAA0D;QACpE,CAAC;QACD,MAAM,gBAAgB,GACpB,SAAS,EAAE,gBAAgB;YAC3B,CAAC,CAAC,MAAoC,EAAE,mBAA4C,EAAE,EAAE,CACtF,IAAI,wBAAwB,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,MAAM,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACnC,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,EAAE;gBAC1C,GAAG,SAAS,EAAE,mBAAmB;gBACjC,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;oBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;wBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,oFAAoF;IACpF,2BAA2B;QACzB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACrC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;aAClD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,YAAY,CACV,QAAsC,EACtC,gBAAkF;QAElF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,GAAG,IAAI,CAAC,mBAAmB;YAC3B,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;oBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,IAAY,EAAE,KAAc;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,cAAc,GAAG,KAAK,IAAI,SAAS,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,IAAY;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;IACtC,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrD,IAAI,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3E,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB,EAAE,IAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,GAAG,YAAY,8BAA8B,CAAC;wBAAE,MAAM,GAAG,CAAC;oBAChE,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,qBAAqB,CACxB,UAAU,EACV,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,cAAc,EACrB,IAAI,8BAA8B,CAAC,gCAAgC,CAAC,CACrE,CAAC;YACJ,CAAC;YACD,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,MAAM,uBAAuB,UAAU,iBAAiB,CACpF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA4F;QAE5F,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,uEAAuE;YACvE,6DAA6D;YAC7D,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,8BAA8B,EAAE,CAAC;gBAClD,qEAAqE;gBACrE,qEAAqE;gBACrE,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,IAAI,iCAAiC,MAAM,CAAC,UAAU,+BAA+B,CACjH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA6B,EAC7B,OAAe,EACf,OAAO,GAA2D,EAAE;QAEpE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,wBAAwB,CAAC,gCAAgC,CAAC,CAAC;QAClF,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,uBAAuB,MAAM,CAAC,UAAU,uBAAuB,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3F,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAA6B;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,OAAO,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,QAA0D,CAAC;YAC/D,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,iDAAiD;YAC7D,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACvD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,GAAG,CAAC,OAAO,CACT,uCAAuC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,GAAG,CACxF,CAAC;oBACF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;oBACnF,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI;oBAAE,SAAS;gBAC5C,GAAG,CAAC,OAAO,CACT,oCAAoC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,IAAI,CACpH,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;wBAC3B,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;wBAC9B,SAAS,EAAE,CAAC;qBACb,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,UAAU,CACZ,iCAAiC,OAAO,CAAC,UAAU,SAAS,MAAM,CAAC,IAAI,GAAG,EAC1E,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,MAAmB,EACnB,UAAkB,EAClB,IAAyB;QAEzB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3E,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACK,qBAAqB,CAAC,MAAmB;QAC/C,IAAI,CAAC,MAAM,CAAC,cAAc;YAAE,OAAO;QACnC,MAAM,IAAI,YAAY,CACpB,2BAA2B,MAAM,CAAC,IAAI,sCAAsC;YAC1E,GAAG,MAAM,CAAC,cAAc,0EAA0E,CACrG,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAC3B,UAAkB,EAClB,UAAkB,EAClB,cAAsB,EACtB,KAAY;QAEZ,MAAM,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,WAAW,IAAI,CAAC;YAAE,OAAO;QAC7B,yEAAyE;QACzE,qEAAqE;QACrE,wDAAwD;QACxD,MAAM,IAAI,YAAY,CACpB,2BAA2B,UAAU,qBAAqB,KAAK,CAAC,OAAO,KAAK;YAC1E,aAAa,UAAU,kBAAkB,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,8BAA8B,CACvG,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/C,SAAS,CAAC;YACR,IAAI,IAAuD,CAAC;YAC5D,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC7B,QAAQ,IAAI,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;oBAC5D,SAAS;gBACX,CAAC;gBACD,IAAI,MAAc,CAAC;gBACnB,IAAI,cAAkC,CAAC;gBACvC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;oBAChD,MAAM,GAAG,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/E,cAAc;wBACZ,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,IAAI,MAAM,CAAC,cAAc;4BAChE,CAAC,CAAC,MAAM,CAAC,cAAc;4BACvB,CAAC,CAAC,SAAS,CAAC;gBAClB,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;gBACD,SAAS,IAAI,CAAC,CAAC;gBACf,IAAI,cAAc,EAAE,CAAC;oBACnB,0DAA0D;oBAC1D,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,MAAM,cAAc,EAAE,CAAC,CAAC;oBACrD,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACxC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBACvC,IAAI,IAAI,CAAC,CAAC;oBACV,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBACjF,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI;oBAAE,IAAI,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACzD,CAAC;YACD,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC,MAAM,CAAC;YAC7B,IAAI,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChF,MAAM,IAAI,YAAY,CACpB,4EAA4E,MAAM,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,IAAI,YAAY,CACpB,8DAA8D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CACtG,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,UAA8B;QAC9C,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,mBAAmB,EAAE,CAAC","sourcesContent":["import * as log from \"../log.js\";\nimport type { GondolinRemoteSettings, GondolinRemoteWorkerSettings } from \"../types.js\";\nimport { SandboxError } from \"./errors.js\";\nimport { gondolinPlacements, type GondolinPlacementStore } from \"./gondolin-placement.js\";\nimport {\n GondolinRemoteConnection,\n GondolinWorkerUnreachableError,\n type GondolinRemoteOverrides,\n} from \"./gondolin-remote.js\";\nimport {\n GondolinRuntimeGoneError,\n type GondolinRuntimeHandle,\n type GondolinRuntimeSpec,\n type GondolinRuntimeTransport,\n} from \"./gondolin-worker-client.js\";\nimport type { ExecResult } from \"./types.js\";\n\nconst LEASE_TTL_SECONDS = 300;\nconst QUEUE_WAIT_SECONDS = 60;\nconst QUEUE_POLL_MS = 2000;\n\n/** The connection surface the fleet needs from each worker. */\nexport interface GondolinWorkerConnection {\n health(): Promise<Record<string, unknown>>;\n ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle>;\n stop(\n handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\">,\n ): Promise<void>;\n exec(\n handle: GondolinRuntimeHandle,\n command: string,\n options?: { env?: Record<string, string>; signal?: AbortSignal },\n ): Promise<ExecResult>;\n isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean>;\n listRuntimes(): Promise<Array<{ sessionId: string; instanceId: string }>>;\n dispose(): void;\n}\n\ninterface FleetWorker {\n name: string;\n settings: GondolinRemoteWorkerSettings;\n connection: GondolinWorkerConnection;\n /** Non-empty when the worker's shared workspace failed its usability probe. */\n workspaceError?: string;\n}\n\ninterface GondolinFleetOverrides {\n createConnection?: (\n worker: GondolinRemoteWorkerSettings,\n connectionOverrides: GondolinRemoteOverrides,\n ) => GondolinWorkerConnection;\n placements?: GondolinPlacementStore;\n connectionOverrides?: GondolinRemoteOverrides;\n now?: () => number;\n queuePollMs?: number;\n sleep?: (ms: number) => Promise<void>;\n}\n\nfunction normalizeWorkers(settings: GondolinRemoteSettings): GondolinRemoteWorkerSettings[] {\n const inline = settings.workers ?? [{ url: settings.url as string }];\n return inline.map((worker) => ({\n ...worker,\n name: worker.name ?? worker.url,\n caFile: worker.caFile ?? settings.caFile,\n certFile: worker.certFile ?? settings.certFile,\n keyFile: worker.keyFile ?? settings.keyFile,\n workspaceRoot: worker.workspaceRoot ?? settings.workspaceRoot,\n maxRuntimes: worker.maxRuntimes ?? settings.maxRuntimes,\n }));\n}\n\n/**\n * Places each conversation's runtime on one worker of the fleet and keeps it\n * there (sticky placement). Failover to another worker happens only when the\n * placed worker is unreachable AND its lease fencing watermark has passed —\n * the point after which the old worker's janitor has provably stopped the\n * runtime, so shared storage never sees two writers.\n */\nclass GondolinFleetClient implements GondolinRuntimeTransport {\n private settings?: GondolinRemoteSettings;\n private workers = new Map<string, FleetWorker>();\n private connectionOverrides: GondolinRemoteOverrides = {};\n private placements: GondolinPlacementStore = gondolinPlacements;\n private leaseTtlMs = LEASE_TTL_SECONDS * 1000;\n private queueWaitMs = QUEUE_WAIT_SECONDS * 1000;\n private queuePollMs = QUEUE_POLL_MS;\n private now: () => number = Date.now;\n private sleep: (ms: number) => Promise<void> = (ms) =>\n new Promise((resolve) => setTimeout(resolve, ms));\n\n configure(settings?: GondolinRemoteSettings, overrides?: GondolinFleetOverrides): void {\n for (const worker of this.workers.values()) worker.connection.dispose();\n this.workers.clear();\n this.settings = settings;\n this.placements = overrides?.placements ?? gondolinPlacements;\n this.now = overrides?.now ?? Date.now;\n this.sleep = overrides?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));\n this.queuePollMs = overrides?.queuePollMs ?? QUEUE_POLL_MS;\n this.connectionOverrides = overrides?.connectionOverrides ?? {};\n this.leaseTtlMs = (overrides?.connectionOverrides?.leaseTtlSeconds ?? LEASE_TTL_SECONDS) * 1000;\n this.queueWaitMs = (settings?.queueWaitSeconds ?? QUEUE_WAIT_SECONDS) * 1000;\n if (!settings) return;\n if (!settings.workers?.length && !settings.url) {\n return; // not configured; validate() reports the actionable error\n }\n const createConnection =\n overrides?.createConnection ??\n ((worker: GondolinRemoteWorkerSettings, connectionOverrides: GondolinRemoteOverrides) =>\n new GondolinRemoteConnection(worker, connectionOverrides));\n for (const worker of normalizeWorkers(settings)) {\n const name = worker.name as string;\n const connection = createConnection(worker, {\n ...overrides?.connectionOverrides,\n onLeaseActivity: (instanceId, expiresAtMs) => {\n if (this.placements.get(instanceId)?.worker === name) {\n this.placements.touch(instanceId, expiresAtMs);\n }\n },\n });\n this.workers.set(name, { name, settings: worker, connection });\n }\n }\n\n isConfigured(): boolean {\n return this.workers.size > 0;\n }\n\n /** Names of statically-configured workers with no workspaceRoot (startup guard). */\n workersMissingWorkspaceRoot(): string[] {\n return Array.from(this.workers.values())\n .filter((worker) => !worker.settings.workspaceRoot)\n .map((worker) => worker.name);\n }\n\n /**\n * Attach a dial-home worker at runtime (the gateway calls this when a\n * worker registers). The factory receives the fleet's connection overrides\n * plus the lease-watermark callback, mirroring configure()'s wiring.\n */\n attachWorker(\n settings: GondolinRemoteWorkerSettings,\n createConnection: (overrides: GondolinRemoteOverrides) => GondolinWorkerConnection,\n ): void {\n const name = settings.name ?? settings.url;\n this.workers.get(name)?.connection.dispose();\n const connection = createConnection({\n ...this.connectionOverrides,\n onLeaseActivity: (instanceId, expiresAtMs) => {\n if (this.placements.get(instanceId)?.worker === name) {\n this.placements.touch(instanceId, expiresAtMs);\n }\n },\n });\n this.workers.set(name, { name, settings: { ...settings, name }, connection });\n }\n\n /**\n * Record a worker's shared-workspace health (the gateway pushes this from\n * register and ping frames). A degraded worker keeps its runtimes and its\n * connection, but new placements skip it and runtime operations fail fast\n * with the probe's diagnosis instead of hanging on a dead mount.\n */\n setWorkspaceHealth(name: string, error?: string): void {\n const worker = this.workers.get(name);\n if (!worker) return;\n worker.workspaceError = error || undefined;\n }\n\n /**\n * Detach a dial-home worker (disconnect). Placements survive: their lease\n * watermarks keep fencing failover until expiry, exactly as for an\n * unreachable static worker.\n */\n detachWorker(name: string): void {\n const worker = this.workers.get(name);\n if (!worker) return;\n worker.connection.dispose();\n this.workers.delete(name);\n }\n\n imageSelector(): string | undefined {\n return this.settings?.imageSelector;\n }\n\n /** At least one worker must answer for the sandbox mode to come up. */\n async health(): Promise<Array<{ worker: string; health?: Record<string, unknown> }>> {\n const report = await Promise.all(\n Array.from(this.workers.values()).map(async (worker) => {\n try {\n return { worker: worker.name, health: await worker.connection.health() };\n } catch {\n return { worker: worker.name };\n }\n }),\n );\n if (report.length > 0 && !report.some((entry) => entry.health)) {\n throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n }\n return report;\n }\n\n async ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle> {\n const placed = this.placements.get(instanceId);\n if (placed) {\n const worker = this.workers.get(placed.worker);\n if (worker) {\n try {\n return await this.ensureOn(worker, instanceId, spec);\n } catch (err) {\n if (!(err instanceof GondolinWorkerUnreachableError)) throw err;\n this.assertFailoverAllowed(instanceId, placed.worker, placed.leaseExpiresAt, err);\n }\n } else {\n this.assertFailoverAllowed(\n instanceId,\n placed.worker,\n placed.leaseExpiresAt,\n new GondolinWorkerUnreachableError(\"worker is no longer configured\"),\n );\n }\n log.logWarning(\n `Gondolin worker '${placed.worker}' lost runtime for '${instanceId}'; failing over`,\n );\n }\n const worker = await this.selectWorker();\n return this.ensureOn(worker, instanceId, spec);\n }\n\n async stop(\n handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\" | \"workerName\">,\n ): Promise<void> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) {\n // the worker left the fleet config; its janitor stops the runtime when\n // the lease expires, and the record keeps fencing until then\n return;\n }\n try {\n await worker.connection.stop(handle);\n this.placements.delete(handle.instanceId);\n } catch (err) {\n if (err instanceof GondolinWorkerUnreachableError) {\n // cannot confirm the stop; keep the placement so its watermark still\n // fences any failover, and let the daemon's expiry janitor finish it\n log.logWarning(\n `Gondolin worker '${worker.name}' unreachable while stopping '${handle.instanceId}'; leaving it to lease expiry`,\n );\n return;\n }\n throw err;\n }\n }\n\n async exec(\n handle: GondolinRuntimeHandle,\n command: string,\n options: { env?: Record<string, string>; signal?: AbortSignal } = {},\n ): Promise<ExecResult> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) throw new GondolinRuntimeGoneError(\"worker is no longer configured\");\n this.assertWorkspaceUsable(worker);\n log.logInfo(`Gondolin execution '${handle.instanceId}' routed to worker '${worker.name}'`);\n return worker.connection.exec(handle, command, options);\n }\n\n async isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean> {\n const worker = this.workerFor(handle.workerName);\n if (!worker) return false;\n return worker.connection.isRuntimeAlive(handle);\n }\n\n /**\n * Fleet reconciliation: a runtime whose conversation is placed on a\n * different worker is a stray from a superseded placement — stop it. A\n * runtime with no placement at all (crash between daemon ensure and the\n * placement write) is adopted instead of killed.\n */\n async reconcile(): Promise<void> {\n for (const worker of this.workers.values()) {\n let runtimes: Array<{ sessionId: string; instanceId: string }>;\n try {\n runtimes = await worker.connection.listRuntimes();\n } catch {\n continue; // unreachable workers reconcile on a later sweep\n }\n for (const runtime of runtimes) {\n const record = this.placements.get(runtime.instanceId);\n if (!record) {\n log.logInfo(\n `Adopting unplaced Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}'`,\n );\n this.placements.set(runtime.instanceId, worker.name, this.now() + this.leaseTtlMs);\n continue;\n }\n if (record.worker === worker.name) continue;\n log.logInfo(\n `Stopping stray Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}' (placed on '${record.worker}')`,\n );\n try {\n await worker.connection.stop({\n sessionId: runtime.sessionId,\n instanceId: runtime.instanceId,\n workerPid: 0,\n });\n } catch (err) {\n log.logWarning(\n `Failed to stop stray runtime '${runtime.instanceId}' on '${worker.name}'`,\n err instanceof Error ? err.message : String(err),\n );\n }\n }\n }\n }\n\n private async ensureOn(\n worker: FleetWorker,\n instanceId: string,\n spec: GondolinRuntimeSpec,\n ): Promise<GondolinRuntimeHandle> {\n this.assertWorkspaceUsable(worker);\n const handle = await worker.connection.ensure(instanceId, spec);\n this.placements.set(instanceId, worker.name, this.now() + this.leaseTtlMs);\n return { ...handle, workerName: worker.name };\n }\n\n /**\n * Fail fast — with the probe's diagnosis — instead of dispatching work that\n * would hang on a dead shared-workspace mount. Deliberately NOT a\n * GondolinWorkerUnreachableError: a broken mount must not trigger failover,\n * because the workspace is shared and likely broken for every worker.\n */\n private assertWorkspaceUsable(worker: FleetWorker): void {\n if (!worker.workspaceError) return;\n throw new SandboxError(\n `Error: Gondolin worker '${worker.name}' has an unusable shared workspace: ` +\n `${worker.workspaceError} (fix the mount on the worker; health refreshes with its next heartbeat)`,\n );\n }\n\n private assertFailoverAllowed(\n instanceId: string,\n workerName: string,\n leaseExpiresAt: number,\n cause: Error,\n ): void {\n const remainingMs = leaseExpiresAt - this.now();\n if (remainingMs <= 0) return;\n // Failing over earlier could put a second writable VM on the same shared\n // workspace: the old worker may be alive behind a partition, and its\n // daemon only stops the runtime once the lease expires.\n throw new SandboxError(\n `Error: Gondolin worker '${workerName}' is unreachable (${cause.message}); ` +\n `retrying '${instanceId}' elsewhere in ${Math.ceil(remainingMs / 1000)}s once its lease has expired`,\n );\n }\n\n private async selectWorker(): Promise<FleetWorker> {\n const deadline = this.now() + this.queueWaitMs;\n for (;;) {\n let best: { worker: FleetWorker; load: number } | undefined;\n let reachable = 0;\n let draining = 0;\n let full = 0;\n const degraded: string[] = [];\n for (const worker of this.workers.values()) {\n if (worker.settings.draining) {\n draining += 1;\n continue;\n }\n if (worker.workspaceError) {\n degraded.push(`'${worker.name}': ${worker.workspaceError}`);\n continue;\n }\n let active: number;\n let workspaceError: string | undefined;\n try {\n const health = await worker.connection.health();\n active = typeof health.activeRuntimes === \"number\" ? health.activeRuntimes : 0;\n workspaceError =\n typeof health.workspaceError === \"string\" && health.workspaceError\n ? health.workspaceError\n : undefined;\n } catch {\n continue;\n }\n reachable += 1;\n if (workspaceError) {\n // static workers report probe results via /v1/health only\n degraded.push(`'${worker.name}': ${workspaceError}`);\n continue;\n }\n const cap = worker.settings.maxRuntimes;\n if (cap !== undefined && active >= cap) {\n full += 1;\n continue;\n }\n const load = cap !== undefined ? active / cap : active / Number.MAX_SAFE_INTEGER;\n if (!best || load < best.load) best = { worker, load };\n }\n if (best) return best.worker;\n if (reachable === 0 && draining === 0 && degraded.length === 0) {\n throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n }\n if (full === 0) {\n const detail = degraded.length > 0 ? ` — degraded: ${degraded.join(\"; \")}` : \"\";\n throw new SandboxError(\n `Error: every gondolin remote worker is draining, degraded, or unreachable${detail}`,\n );\n }\n if (this.now() >= deadline) {\n throw new SandboxError(\n `Error: all gondolin remote workers are at capacity (waited ${Math.round(this.queueWaitMs / 1000)}s)`,\n );\n }\n await this.sleep(this.queuePollMs);\n }\n }\n\n private workerFor(workerName: string | undefined): FleetWorker | undefined {\n if (!workerName) return undefined;\n return this.workers.get(workerName);\n }\n}\n\nexport const gondolinFleet = new GondolinFleetClient();\n"]}
@@ -14,6 +14,8 @@ interface GondolinWorkerRegistration {
14
14
  sessionId: string;
15
15
  instanceId: string;
16
16
  }>;
17
+ /** Non-empty when the worker's shared workspace failed its usability probe. */
18
+ workspaceError?: string;
17
19
  }
18
20
  interface GatewayOverrides {
19
21
  /** Server factory (tests substitute a plain net.Server). */
@@ -44,6 +46,7 @@ declare class GondolinWorkerGateway {
44
46
  name: string;
45
47
  connected: boolean;
46
48
  activeRuntimes: number;
49
+ workspaceError?: string;
47
50
  info: GondolinWorkerRegistration;
48
51
  }>;
49
52
  /**
@@ -57,6 +60,13 @@ declare class GondolinWorkerGateway {
57
60
  private handleConnection;
58
61
  private acceptControlChannel;
59
62
  private handleControlFrame;
63
+ /**
64
+ * Track shared-workspace health from register/ping frames and push it into
65
+ * the fleet, logging only the transitions. A worker whose NFS mount died
66
+ * keeps its control channel alive — this is what turns that silent state
67
+ * into refused placements and fail-fast execs.
68
+ */
69
+ private applyWorkspaceHealth;
60
70
  /** RPC to a worker over its control channel (the reverse of HTTPS requests). */
61
71
  private rpc;
62
72
  /** Ask the worker to dial back a data connection for one session tunnel. */
@@ -1 +1 @@
1
- {"version":3,"file":"gondolin-gateway.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-gateway.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAI/C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAc3D,kFAAkF;AAClF,UAAU,0BAA0B;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7D;AAiED,UAAU,gBAAgB;IACxB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC;IAClE,yEAAyE;IACzE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAMD;;;;;;GAMG;AACH,cAAM,qBAAqB;IACzB,OAAO,CAAC,QAAQ,CAAC,CAA0B;IAC3C,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAG3B;IAEJ,SAAS,CAAC,QAAQ,CAAC,EAAE,uBAAuB,EAAE,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAIhF;IAED,YAAY,IAAI,OAAO,CAEtB;IAED,2EAA2E;IAC3E,IAAI,IAAI,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,OAAO,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,IAAI,EAAE,0BAA0B,CAAC;KAClC,CAAC,CAOD;IAED;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAqC3B;IAED,IAAI,IAAI,IAAI,CAUX;IAED,2EAA2E;IAC3E,OAAO,CAAC,gBAAgB;IA6BxB,OAAO,CAAC,oBAAoB;IAgE5B,OAAO,CAAC,kBAAkB;IAoB1B,gFAAgF;IAChF,OAAO,CAAC,GAAG;IAoCX,4EAA4E;IAC5E,OAAO,CAAC,UAAU;IAyDlB;;;;OAIG;YACW,UAAU;IA0BxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,cAAc;CAMvB;AAED,eAAO,MAAM,eAAe,uBAA8B,CAAC"}
1
+ {"version":3,"file":"gondolin-gateway.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-gateway.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAI/C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAc3D,kFAAkF;AAClF,UAAU,0BAA0B;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAkED,UAAU,gBAAgB;IACxB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC;IAClE,yEAAyE;IACzE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAMD;;;;;;GAMG;AACH,cAAM,qBAAqB;IACzB,OAAO,CAAC,QAAQ,CAAC,CAA0B;IAC3C,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAG3B;IAEJ,SAAS,CAAC,QAAQ,CAAC,EAAE,uBAAuB,EAAE,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAIhF;IAED,YAAY,IAAI,OAAO,CAEtB;IAED,2EAA2E;IAC3E,IAAI,IAAI,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,OAAO,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,IAAI,EAAE,0BAA0B,CAAC;KAClC,CAAC,CAQD;IAED;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6C3B;IAED,IAAI,IAAI,IAAI,CAUX;IAED,2EAA2E;IAC3E,OAAO,CAAC,gBAAgB;IA6BxB,OAAO,CAAC,oBAAoB;IAiE5B,OAAO,CAAC,kBAAkB;IAsB1B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAiB5B,gFAAgF;IAChF,OAAO,CAAC,GAAG;IAoCX,4EAA4E;IAC5E,OAAO,CAAC,UAAU;IAyDlB;;;;OAIG;YACW,UAAU;IA0BxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,cAAc;CAMvB;AAED,eAAO,MAAM,eAAe,uBAA8B,CAAC"}
@@ -78,6 +78,7 @@ class GondolinWorkerGateway {
78
78
  name: worker.info.name,
79
79
  connected: !worker.socket.destroyed,
80
80
  activeRuntimes: worker.activeRuntimes,
81
+ workspaceError: worker.workspaceError,
81
82
  info: worker.info,
82
83
  }));
83
84
  }
@@ -90,6 +91,12 @@ class GondolinWorkerGateway {
90
91
  if (!this.settings || this.server)
91
92
  return;
92
93
  const settings = this.settings;
94
+ if (!settings.workspaceRoot) {
95
+ // dial-home workers inherit workspaceRoot from here; without it every
96
+ // runtime is rejected with "escapes the workspace root"
97
+ throw new Error("gondolin gateway requires workspaceRoot (the worker-side shared workspace path); " +
98
+ "set sandbox.gondolin.remote.gateway.workspaceRoot");
99
+ }
93
100
  let material;
94
101
  if (settings.certFile && settings.keyFile && settings.clientCaFile) {
95
102
  material = {
@@ -203,6 +210,7 @@ class GondolinWorkerGateway {
203
210
  request: (method, path, body, headers) => this.rpc(name, method, path, body, headers),
204
211
  tunnel: (path, headers, callbacks) => this.openTunnel(name, path, headers, callbacks),
205
212
  }));
213
+ this.applyWorkspaceHealth(worker, registration.workspaceError);
206
214
  // a worker that reconnected may carry runtimes whose placements moved
207
215
  void gondolinFleet.reconcile();
208
216
  socket.on("close", () => {
@@ -226,6 +234,8 @@ class GondolinWorkerGateway {
226
234
  if (typeof frame.activeRuntimes === "number") {
227
235
  worker.activeRuntimes = frame.activeRuntimes;
228
236
  }
237
+ // omitted on healthy pings (Go omitempty), so absence means recovered
238
+ this.applyWorkspaceHealth(worker, frame.workspaceError);
229
239
  socket.write(encodeControlFrame({ type: "pong" }));
230
240
  return;
231
241
  }
@@ -238,6 +248,27 @@ class GondolinWorkerGateway {
238
248
  pending.resolve(frame);
239
249
  }
240
250
  }
251
+ /**
252
+ * Track shared-workspace health from register/ping frames and push it into
253
+ * the fleet, logging only the transitions. A worker whose NFS mount died
254
+ * keeps its control channel alive — this is what turns that silent state
255
+ * into refused placements and fail-fast execs.
256
+ */
257
+ applyWorkspaceHealth(worker, error) {
258
+ const next = error || undefined;
259
+ if (next === worker.workspaceError) {
260
+ gondolinFleet.setWorkspaceHealth(worker.info.name, next); // attach may have reset it
261
+ return;
262
+ }
263
+ if (next) {
264
+ log.logWarning(`Gondolin worker '${worker.info.name}' workspace is unusable: ${next} — new placements disabled until it recovers`);
265
+ }
266
+ else {
267
+ log.logInfo(`Gondolin worker '${worker.info.name}' workspace recovered`);
268
+ }
269
+ worker.workspaceError = next;
270
+ gondolinFleet.setWorkspaceHealth(worker.info.name, next);
271
+ }
241
272
  /** RPC to a worker over its control channel (the reverse of HTTPS requests). */
242
273
  rpc(name, method, path, body, headers) {
243
274
  const worker = this.workers.get(name);
@@ -1 +1 @@
1
- {"version":3,"file":"gondolin-gateway.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-gateway.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,YAAY,IAAI,eAAe,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,8BAA8B,EAC9B,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,cAAc,GAAG,OAAO,CAAC;AAC/B,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAqCjC;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,OAAwD;IAIxF,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,CAAC,KAAa;YAChB,IAAI,OAAO;gBAAE,OAAO;YACpB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACxC,SAAS,CAAC;gBACR,uEAAuE;gBACvE,IAAI,OAAO,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO;gBACzC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACtC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM;oBAAE,OAAO;gBACvC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC/C,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;gBACrC,IAAI,KAAmB,CAAC;gBACxB,IAAI,CAAC;oBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAiB,CAAC;gBAC/D,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,CAAC,wBAAwB;gBACpC,CAAC;gBACD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI;YACF,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,oBAAoB,CAAC,KAAK,CAAC,CAAC;AACrC,CAAC;AAYD,SAAS,mBAAmB,CAAC,MAAc;IACzC,OAAO,MAAM,YAAY,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC;AAC1D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB;IAA3B;QAEU,cAAS,GAAqB,EAAE,CAAC;QAExB,YAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAC9C,mBAAc,GAAG,IAAI,GAAG,EAGtC,CAAC;IAoVN,CAAC;IAlVC,SAAS,CAAC,QAAkC,EAAE,SAA4B;QACxE,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,CAAC;IACnC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;IACrC,CAAC;IAED,2EAA2E;IAC3E,IAAI;QAMF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACxD,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI;YACtB,SAAS,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,QAAqE,CAAC;QAC1E,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YACnE,QAAQ,GAAG;gBACT,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;aACpC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACvF,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GACX,IAAI,CAAC,SAAS,CAAC,YAAY;YAC3B,CAAC,CAAC,YAAsC,EAAE,EAAE,CAC1C,eAAe,CACb;gBACE,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACrC,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACnC,EAAE,EAAE,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACvC,WAAW,EAAE,IAAI;gBACjB,8DAA8D;gBAC9D,mDAAmD;gBACnD,kBAAkB,EAAE,KAAK;gBACzB,UAAU,EAAE,SAAS;aACtB,EACD,YAAY,CACb,CAAC,CAAC;QACP,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;YAC1C,GAAG,CAAC,OAAO,CAAC,yCAAyC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE;gBAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChF,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED,2EAA2E;IACnE,gBAAgB,CAAC,MAAc;QACrC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,wBAAwB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,CAAC;gBACd,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC1B,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACtC,CAAC;qBAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBACvB,iEAAiE;oBACjE,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnD,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBACnD,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAClD,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;oBAClC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;gBACD,OAAO;YACT,CAAC;YACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,CAAC;IAEO,oBAAoB,CAC1B,MAAc,EACd,YAA0B,EAC1B,OAA6B;QAE7B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAc,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,kDAAkD,CAAC,CAAC;YACxF,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,MAAM,GAAqB;YAC/B,IAAI,EAAE,YAAY;YAClB,MAAM;YACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;YACpB,cAAc,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YAClD,UAAU,EAAE,IAAI,GAAG,EAAE;YACrB,SAAS,EAAE,CAAC;SACb,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CACT,2BAA2B,IAAI,KAAK,YAAY,CAAC,EAAE,IAAI,YAAY,CAAC,IAAI,IAAI;YAC1E,GAAG,YAAY,CAAC,WAAW,KAAK,YAAY,CAAC,IAAI,SAAS;YAC1D,OAAO,YAAY,CAAC,WAAW,IAAI,GAAG,KAAK,SAAS,sBAAsB,CAC7E,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;QACjD,aAAa,CAAC,YAAY,CACxB;YACE,IAAI;YACJ,GAAG,EAAE,cAAc,IAAI,EAAE;YACzB,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa;YAC3C,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,YAAY,CAAC,WAAW;YAC/D,QAAQ,EAAE,SAAS,EAAE,QAAQ;SAC9B,EACD,CAAC,mBAAmB,EAAE,EAAE,CACtB,IAAI,wBAAwB,CAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,cAAc,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,EAAE,EAChF;YACE,GAAG,mBAAmB;YACtB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,KAAK,SAAS;gBAC9C,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;YACrF,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;SACtF,CACF,CACJ,CAAC;QACF,sEAAsE;QACtE,KAAK,aAAa,CAAC,SAAS,EAAE,CAAC;QAE/B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,OAAO,KAAK,MAAM;gBAAE,OAAO,CAAC,aAAa;YAC7C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE;gBAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC9E,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAC1B,GAAG,CAAC,UAAU,CAAC,iCAAiC,IAAI,wCAAwC,CAAC,CAAC;YAC9F,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB,CAAC,MAAc,EAAE,KAAmB;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;gBAC7C,MAAM,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YAC/C,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,gFAAgF;IACxE,GAAG,CACT,IAAY,EACZ,MAAc,EACd,IAAY,EACZ,IAAa,EACb,OAAgC;QAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,8BAA8B,CAAC,WAAW,IAAI,oBAAoB,CAAC,CACxE,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7B,MAAM,CACJ,IAAI,8BAA8B,CAChC,kBAAkB,IAAI,gBAAgB,MAAM,IAAI,IAAI,GAAG,CACxD,CACF,CAAC;YACJ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,cAAc,CAAC,CAAC;YAClD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE;gBACxB,KAAK;gBACL,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CACjB,OAAO,CAAC;oBACN,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;oBACzB,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAA4B;iBACpD,CAAC;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAChG,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IACpE,UAAU,CAChB,IAAY,EACZ,IAAY,EACZ,OAA+B,EAC/B,SAAiC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,WAAW,IAAI,oBAAoB,CAA0B,CAAC;YACtF,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,sCAAsC;YACnE,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClC,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,WAAW,IAAI,8BAA8B,CACrB,CAAC;gBAC3B,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,IAAI,iBAAiB,CAAC,CAAC;YACxD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE;gBAC7B,KAAK;gBACL,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;oBAChC,IAAI,MAAM,GAAG,KAAK,CAAC;oBACnB,MAAM,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBAClD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;wBAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACzC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;wBAClC,IAAI,MAAM;4BAAE,OAAO;wBACnB,MAAM,GAAG,IAAI,CAAC;wBACd,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC,CAAC,CAAC;oBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;wBACtB,IAAI,MAAM;4BAAE,OAAO;wBACnB,MAAM,GAAG,IAAI,CAAC;wBACd,SAAS,CAAC,OAAO,EAAE,CAAC;oBACtB,CAAC,CAAC,CAAC;oBACH,OAAO,CAAC;wBACN,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;4BAChB,IAAI,CAAC,MAAM;gCAAE,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC3D,CAAC;wBACD,KAAK,EAAE,GAAG,EAAE;4BACV,IAAI,MAAM;gCAAE,OAAO;4BACnB,MAAM,GAAG,IAAI,CAAC;4BACd,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,CAAC;qBACF,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU,CAAC,MAAc,EAAE,KAAmB;QAC1D,MAAM,MAAM,GAAG,CAAC,OAAe,EAAQ,EAAE;YACvC,GAAG,CAAC,UAAU,CAAC,iCAAiC,OAAO,EAAE,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;YACjC,MAAM,CAAC,qCAAqC,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC9C,MAAM,CAAC,oCAAoC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,0CAA0C,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YACjE,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,oBAAoB,KAAK,CAAC,IAAI,qCAAqC,CAAC,CAAC;YACjF,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,KAAa,EAAE,QAAgB;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,cAAc,CAAC,MAAc;QACnC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9C,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,qBAAqB,EAAE,CAAC","sourcesContent":["import { randomBytes } from \"node:crypto\";\nimport { readFileSync } from \"node:fs\";\nimport type { Server, Socket } from \"node:net\";\nimport { hostname } from \"node:os\";\nimport { TLSSocket, createServer as createTlsServer } from \"node:tls\";\nimport * as log from \"../log.js\";\nimport type { GondolinGatewaySettings } from \"../types.js\";\nimport { gondolinFleet } from \"./gondolin-fleet.js\";\nimport { gondolinJoin } from \"./gondolin-join.js\";\nimport {\n GondolinRemoteConnection,\n GondolinWorkerUnreachableError,\n createSessionFrameParser,\n encodeSessionMessage,\n} from \"./gondolin-remote.js\";\nimport type { SessionClient, SessionClientCallbacks } from \"./gondolin-worker-client.js\";\n\nconst RPC_TIMEOUT_MS = 150_000;\nconst TUNNEL_TIMEOUT_MS = 15_000;\n\n/** What a worker announces when it dials home (mirrors the Go register frame). */\ninterface GondolinWorkerRegistration {\n name: string;\n os?: string;\n arch?: string;\n accelerator?: string;\n cpus?: number;\n memoryBytes?: number;\n maxRuntimes?: number;\n protocolVersion?: number;\n runtimes?: Array<{ sessionId: string; instanceId: string }>;\n}\n\ninterface ControlFrame extends GondolinWorkerRegistration {\n type: string;\n id?: number;\n status?: number;\n body?: unknown;\n nonce?: string;\n activeRuntimes?: number;\n /** join exchange */\n token?: string;\n csr?: string;\n}\n\ninterface RegisteredWorker {\n info: GondolinWorkerRegistration;\n socket: Socket;\n connectedAt: number;\n lastSeen: number;\n activeRuntimes: number;\n pendingRpc: Map<number, { resolve: (frame: ControlFrame) => void; timer: NodeJS.Timeout }>;\n nextRpcId: number;\n}\n\n/**\n * Splits a byte stream into u32be-length-prefixed JSON frames, preserving any\n * bytes after the point where the consumer stops parsing (a tunnel connection\n * switches protocols after its preamble frame).\n */\nfunction createControlFrameReader(onFrame: (frame: ControlFrame, leftover: Buffer) => void): {\n push: (chunk: Buffer) => void;\n stop: () => void;\n} {\n let buffer = Buffer.alloc(0);\n let stopped = false;\n return {\n push(chunk: Buffer) {\n if (stopped) return;\n buffer = Buffer.concat([buffer, chunk]);\n for (;;) {\n // onFrame may call stop() synchronously (a tunnel switching protocols)\n if (stopped || buffer.length < 4) return;\n const length = buffer.readUInt32BE(0);\n if (buffer.length < 4 + length) return;\n const payload = buffer.subarray(4, 4 + length);\n buffer = buffer.subarray(4 + length);\n let frame: ControlFrame;\n try {\n frame = JSON.parse(payload.toString(\"utf8\")) as ControlFrame;\n } catch {\n continue; // skip malformed frames\n }\n onFrame(frame, buffer);\n }\n },\n stop() {\n stopped = true;\n },\n };\n}\n\nfunction encodeControlFrame(frame: object): Buffer {\n return encodeSessionMessage(frame);\n}\n\ninterface GatewayOverrides {\n /** Server factory (tests substitute a plain net.Server). */\n createServer?: (onConnection: (socket: Socket) => void) => Server;\n /** mTLS verification result for a socket (tests substitute outcomes). */\n isAuthorized?: (socket: Socket) => boolean;\n leaseTtlSeconds?: number;\n rpcTimeoutMs?: number;\n tunnelTimeoutMs?: number;\n}\n\nfunction defaultIsAuthorized(socket: Socket): boolean {\n return socket instanceof TLSSocket && socket.authorized;\n}\n\n/**\n * The host side of dial-home workers: one mTLS listener that accepts worker\n * control channels (register/ping/RPC responses) and per-command dial-back\n * tunnels, keeps the registry of connected workers, and attaches each one to\n * the fleet as a regular worker connection. Placement, leases, fencing, and\n * failover are unchanged — this only reverses who dials whom.\n */\nclass GondolinWorkerGateway {\n private settings?: GondolinGatewaySettings;\n private overrides: GatewayOverrides = {};\n private server?: Server;\n private readonly workers = new Map<string, RegisteredWorker>();\n private readonly pendingTunnels = new Map<\n string,\n { resolve: (value: { socket: Socket; leftover: Buffer }) => void; timer: NodeJS.Timeout }\n >();\n\n configure(settings?: GondolinGatewaySettings, overrides?: GatewayOverrides): void {\n this.stop();\n this.settings = settings;\n this.overrides = overrides ?? {};\n }\n\n isConfigured(): boolean {\n return this.settings !== undefined;\n }\n\n /** Registered workers and their liveness (admin/observability surface). */\n list(): Array<{\n name: string;\n connected: boolean;\n activeRuntimes: number;\n info: GondolinWorkerRegistration;\n }> {\n return Array.from(this.workers.values()).map((worker) => ({\n name: worker.info.name,\n connected: !worker.socket.destroyed,\n activeRuntimes: worker.activeRuntimes,\n info: worker.info,\n }));\n }\n\n /**\n * Start the listener. Without explicit certificate settings the gateway\n * provisions its own: a private worker CA plus a server certificate signed\n * by it (required for join's CA pinning to also authenticate the host).\n */\n async start(): Promise<void> {\n if (!this.settings || this.server) return;\n const settings = this.settings;\n let material: { certFile: string; keyFile: string; clientCaFile: string };\n if (settings.certFile && settings.keyFile && settings.clientCaFile) {\n material = {\n certFile: settings.certFile,\n keyFile: settings.keyFile,\n clientCaFile: settings.clientCaFile,\n };\n } else {\n if (!gondolinJoin.isConfigured()) {\n throw new Error(\"gondolin gateway auto-init requires the join service to be configured\");\n }\n const issued = await gondolinJoin.ensureServerCert(settings.hostnames ?? [hostname()]);\n material = { ...issued, clientCaFile: gondolinJoin.paths().caFile };\n }\n const factory =\n this.overrides.createServer ??\n ((onConnection: (socket: Socket) => void) =>\n createTlsServer(\n {\n cert: readFileSync(material.certFile),\n key: readFileSync(material.keyFile),\n ca: readFileSync(material.clientCaFile),\n requestCert: true,\n // unauthenticated connections may only send a join frame; the\n // per-frame authorization gate enforces that below\n rejectUnauthorized: false,\n minVersion: \"TLSv1.3\",\n },\n onConnection,\n ));\n this.server = factory((socket) => this.handleConnection(socket));\n this.server.listen(this.settings.port, () => {\n log.logInfo(`Gondolin worker gateway listening on :${this.settings?.port}`);\n });\n }\n\n stop(): void {\n for (const worker of this.workers.values()) {\n worker.socket.destroy();\n for (const pending of worker.pendingRpc.values()) clearTimeout(pending.timer);\n }\n this.workers.clear();\n for (const pending of this.pendingTunnels.values()) clearTimeout(pending.timer);\n this.pendingTunnels.clear();\n this.server?.close();\n this.server = undefined;\n }\n\n /** Routes one inbound connection by its first frame: control or tunnel. */\n private handleConnection(socket: Socket): void {\n socket.setNoDelay(true);\n let routed = false;\n const authorized = (this.overrides.isAuthorized ?? defaultIsAuthorized)(socket);\n const reader = createControlFrameReader((frame, leftover) => {\n if (!routed) {\n routed = true;\n if (frame.type === \"join\") {\n void this.handleJoin(socket, frame);\n } else if (!authorized) {\n // everything except join requires a CA-signed client certificate\n socket.destroy();\n } else if (frame.type === \"register\" && frame.name) {\n this.acceptControlChannel(socket, frame, reader);\n } else if (frame.type === \"tunnel\" && frame.nonce) {\n reader.stop();\n socket.removeAllListeners(\"data\");\n this.acceptTunnel(socket, frame.nonce, leftover);\n } else {\n socket.destroy();\n }\n return;\n }\n this.handleControlFrame(socket, frame);\n });\n socket.on(\"data\", (chunk: Buffer) => reader.push(chunk));\n socket.on(\"error\", () => socket.destroy());\n }\n\n private acceptControlChannel(\n socket: Socket,\n registration: ControlFrame,\n _reader: { stop: () => void },\n ): void {\n const name = registration.name as string;\n const previous = this.workers.get(name);\n if (previous && !previous.socket.destroyed) {\n log.logInfo(`Gondolin worker '${name}' re-registered; superseding previous connection`);\n previous.socket.destroy();\n }\n const worker: RegisteredWorker = {\n info: registration,\n socket,\n connectedAt: Date.now(),\n lastSeen: Date.now(),\n activeRuntimes: registration.runtimes?.length ?? 0,\n pendingRpc: new Map(),\n nextRpcId: 1,\n };\n this.workers.set(name, worker);\n const surviving = registration.runtimes?.length ?? 0;\n log.logInfo(\n `Gondolin worker joined: ${name} (${registration.os}/${registration.arch}, ` +\n `${registration.accelerator}, ${registration.cpus} cpus, ` +\n `cap ${registration.maxRuntimes ?? \"∞\"}, ${surviving} runtimes surviving)`,\n );\n\n const overrides = this.settings?.workers?.[name];\n gondolinFleet.attachWorker(\n {\n name,\n url: `dialhome://${name}`,\n workspaceRoot: this.settings?.workspaceRoot,\n maxRuntimes: overrides?.maxRuntimes ?? registration.maxRuntimes,\n draining: overrides?.draining,\n },\n (connectionOverrides) =>\n new GondolinRemoteConnection(\n { name, url: `dialhome://${name}`, workspaceRoot: this.settings?.workspaceRoot },\n {\n ...connectionOverrides,\n ...(this.overrides.leaseTtlSeconds !== undefined\n ? { leaseTtlSeconds: this.overrides.leaseTtlSeconds }\n : {}),\n request: (method, path, body, headers) => this.rpc(name, method, path, body, headers),\n tunnel: (path, headers, callbacks) => this.openTunnel(name, path, headers, callbacks),\n },\n ),\n );\n // a worker that reconnected may carry runtimes whose placements moved\n void gondolinFleet.reconcile();\n\n socket.on(\"close\", () => {\n const current = this.workers.get(name);\n if (current !== worker) return; // superseded\n for (const pending of worker.pendingRpc.values()) clearTimeout(pending.timer);\n worker.pendingRpc.clear();\n log.logWarning(`Gondolin worker disconnected: ${name} (placements fence until lease expiry)`);\n gondolinFleet.detachWorker(name);\n this.workers.delete(name);\n });\n }\n\n private handleControlFrame(socket: Socket, frame: ControlFrame): void {\n const worker = this.workerBySocket(socket);\n if (!worker) return;\n worker.lastSeen = Date.now();\n if (frame.type === \"ping\") {\n if (typeof frame.activeRuntimes === \"number\") {\n worker.activeRuntimes = frame.activeRuntimes;\n }\n socket.write(encodeControlFrame({ type: \"pong\" }));\n return;\n }\n if (frame.type === \"response\" && typeof frame.id === \"number\") {\n const pending = worker.pendingRpc.get(frame.id);\n if (!pending) return;\n worker.pendingRpc.delete(frame.id);\n clearTimeout(pending.timer);\n pending.resolve(frame);\n }\n }\n\n /** RPC to a worker over its control channel (the reverse of HTTPS requests). */\n private rpc(\n name: string,\n method: string,\n path: string,\n body?: object,\n headers?: Record<string, string>,\n ): Promise<{ status: number; json: Record<string, unknown> }> {\n const worker = this.workers.get(name);\n if (!worker || worker.socket.destroyed) {\n return Promise.reject(\n new GondolinWorkerUnreachableError(`worker '${name}' is not connected`),\n );\n }\n const id = worker.nextRpcId++;\n return new Promise((resolve, reject) => {\n const timer = setTimeout(() => {\n worker.pendingRpc.delete(id);\n reject(\n new GondolinWorkerUnreachableError(\n `RPC to worker '${name}' timed out (${method} ${path})`,\n ),\n );\n }, this.overrides.rpcTimeoutMs ?? RPC_TIMEOUT_MS);\n timer.unref?.();\n worker.pendingRpc.set(id, {\n timer,\n resolve: (frame) =>\n resolve({\n status: frame.status ?? 0,\n json: (frame.body ?? {}) as Record<string, unknown>,\n }),\n });\n worker.socket.write(encodeControlFrame({ type: \"request\", id, method, path, body, headers }));\n });\n }\n\n /** Ask the worker to dial back a data connection for one session tunnel. */\n private openTunnel(\n name: string,\n path: string,\n headers: Record<string, string>,\n callbacks: SessionClientCallbacks,\n ): Promise<SessionClient> {\n const worker = this.workers.get(name);\n if (!worker || worker.socket.destroyed) {\n const error = new Error(`worker '${name}' is not connected`) as NodeJS.ErrnoException;\n error.code = \"ECONNREFUSED\"; // nothing executed — safe to recreate\n return Promise.reject(error);\n }\n const sessionId = path.split(\"/\")[3];\n const nonce = randomBytes(16).toString(\"hex\");\n return new Promise((resolve, reject) => {\n const timer = setTimeout(() => {\n this.pendingTunnels.delete(nonce);\n const error = new Error(\n `worker '${name}' did not dial back a tunnel`,\n ) as NodeJS.ErrnoException;\n error.code = \"ECONNREFUSED\";\n reject(error);\n }, this.overrides.tunnelTimeoutMs ?? TUNNEL_TIMEOUT_MS);\n timer.unref?.();\n this.pendingTunnels.set(nonce, {\n timer,\n resolve: ({ socket, leftover }) => {\n let closed = false;\n const parse = createSessionFrameParser(callbacks);\n if (leftover.length > 0) parse(leftover);\n socket.on(\"data\", parse);\n socket.on(\"error\", (error: Error) => {\n if (closed) return;\n closed = true;\n callbacks.onClose(error);\n });\n socket.on(\"close\", () => {\n if (closed) return;\n closed = true;\n callbacks.onClose();\n });\n resolve({\n send: (message) => {\n if (!closed) socket.write(encodeSessionMessage(message));\n },\n close: () => {\n if (closed) return;\n closed = true;\n socket.destroy();\n },\n });\n },\n });\n worker.socket.write(encodeControlFrame({ type: \"open-tunnel\", nonce, sessionId, headers }));\n });\n }\n\n /**\n * One-time token → CA-signed client certificate. The only operation an\n * unauthenticated connection may perform; the connection closes after the\n * response either way.\n */\n private async handleJoin(socket: Socket, frame: ControlFrame): Promise<void> {\n const refuse = (message: string): void => {\n log.logWarning(`Gondolin worker join refused: ${message}`);\n socket.end(encodeControlFrame({ type: \"join-error\", message }));\n };\n if (!gondolinJoin.isConfigured()) {\n refuse(\"join is not enabled on this gateway\");\n return;\n }\n if (!frame.token || !frame.csr || !frame.name) {\n refuse(\"join requires token, csr, and name\");\n return;\n }\n if (!gondolinJoin.consumeToken(frame.token)) {\n refuse(`invalid or expired join token (worker '${frame.name}')`);\n return;\n }\n try {\n const signed = await gondolinJoin.signCsr(frame.csr);\n log.logInfo(`Gondolin worker '${frame.name}' joined: client certificate issued`);\n socket.end(encodeControlFrame({ type: \"join-ok\", cert: signed.certPem, ca: signed.caPem }));\n } catch (err) {\n refuse(`certificate issuance failed: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n\n private acceptTunnel(socket: Socket, nonce: string, leftover: Buffer): void {\n const pending = this.pendingTunnels.get(nonce);\n if (!pending) {\n socket.destroy();\n return;\n }\n this.pendingTunnels.delete(nonce);\n clearTimeout(pending.timer);\n pending.resolve({ socket, leftover: Buffer.from(leftover) });\n }\n\n private workerBySocket(socket: Socket): RegisteredWorker | undefined {\n for (const worker of this.workers.values()) {\n if (worker.socket === socket) return worker;\n }\n return undefined;\n }\n}\n\nexport const gondolinGateway = new GondolinWorkerGateway();\n"]}
1
+ {"version":3,"file":"gondolin-gateway.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-gateway.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,YAAY,IAAI,eAAe,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,8BAA8B,EAC9B,wBAAwB,EACxB,oBAAoB,GACrB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,cAAc,GAAG,OAAO,CAAC;AAC/B,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAwCjC;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,OAAwD;IAIxF,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO;QACL,IAAI,CAAC,KAAa;YAChB,IAAI,OAAO;gBAAE,OAAO;YACpB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACxC,SAAS,CAAC;gBACR,uEAAuE;gBACvE,IAAI,OAAO,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO;gBACzC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACtC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM;oBAAE,OAAO;gBACvC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC/C,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;gBACrC,IAAI,KAAmB,CAAC;gBACxB,IAAI,CAAC;oBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAiB,CAAC;gBAC/D,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,CAAC,wBAAwB;gBACpC,CAAC;gBACD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI;YACF,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,oBAAoB,CAAC,KAAK,CAAC,CAAC;AACrC,CAAC;AAYD,SAAS,mBAAmB,CAAC,MAAc;IACzC,OAAO,MAAM,YAAY,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC;AAC1D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB;IAA3B;QAEU,cAAS,GAAqB,EAAE,CAAC;QAExB,YAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAC9C,mBAAc,GAAG,IAAI,GAAG,EAGtC,CAAC;IAwXN,CAAC;IAtXC,SAAS,CAAC,QAAkC,EAAE,SAA4B;QACxE,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,CAAC;IACnC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;IACrC,CAAC;IAED,2EAA2E;IAC3E,IAAI;QAOF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACxD,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI;YACtB,SAAS,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,sEAAsE;YACtE,wDAAwD;YACxD,MAAM,IAAI,KAAK,CACb,mFAAmF;gBACjF,mDAAmD,CACtD,CAAC;QACJ,CAAC;QACD,IAAI,QAAqE,CAAC;QAC1E,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YACnE,QAAQ,GAAG;gBACT,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;aACpC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACvF,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GACX,IAAI,CAAC,SAAS,CAAC,YAAY;YAC3B,CAAC,CAAC,YAAsC,EAAE,EAAE,CAC1C,eAAe,CACb;gBACE,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACrC,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACnC,EAAE,EAAE,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACvC,WAAW,EAAE,IAAI;gBACjB,8DAA8D;gBAC9D,mDAAmD;gBACnD,kBAAkB,EAAE,KAAK;gBACzB,UAAU,EAAE,SAAS;aACtB,EACD,YAAY,CACb,CAAC,CAAC;QACP,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;YAC1C,GAAG,CAAC,OAAO,CAAC,yCAAyC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE;gBAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChF,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED,2EAA2E;IACnE,gBAAgB,CAAC,MAAc;QACrC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,wBAAwB,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,CAAC;gBACd,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC1B,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACtC,CAAC;qBAAM,IAAI,CAAC,UAAU,EAAE,CAAC;oBACvB,iEAAiE;oBACjE,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnD,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBACnD,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAClD,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;oBAClC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACnD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;gBACD,OAAO;YACT,CAAC;YACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,CAAC;IAEO,oBAAoB,CAC1B,MAAc,EACd,YAA0B,EAC1B,OAA6B;QAE7B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAc,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,kDAAkD,CAAC,CAAC;YACxF,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,MAAM,GAAqB;YAC/B,IAAI,EAAE,YAAY;YAClB,MAAM;YACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;YACpB,cAAc,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YAClD,UAAU,EAAE,IAAI,GAAG,EAAE;YACrB,SAAS,EAAE,CAAC;SACb,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CACT,2BAA2B,IAAI,KAAK,YAAY,CAAC,EAAE,IAAI,YAAY,CAAC,IAAI,IAAI;YAC1E,GAAG,YAAY,CAAC,WAAW,KAAK,YAAY,CAAC,IAAI,SAAS;YAC1D,OAAO,YAAY,CAAC,WAAW,IAAI,GAAG,KAAK,SAAS,sBAAsB,CAC7E,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;QACjD,aAAa,CAAC,YAAY,CACxB;YACE,IAAI;YACJ,GAAG,EAAE,cAAc,IAAI,EAAE;YACzB,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa;YAC3C,WAAW,EAAE,SAAS,EAAE,WAAW,IAAI,YAAY,CAAC,WAAW;YAC/D,QAAQ,EAAE,SAAS,EAAE,QAAQ;SAC9B,EACD,CAAC,mBAAmB,EAAE,EAAE,CACtB,IAAI,wBAAwB,CAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,cAAc,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,EAAE,EAChF;YACE,GAAG,mBAAmB;YACtB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,KAAK,SAAS;gBAC9C,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;YACrF,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;SACtF,CACF,CACJ,CAAC;QACF,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;QAC/D,sEAAsE;QACtE,KAAK,aAAa,CAAC,SAAS,EAAE,CAAC;QAE/B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,OAAO,KAAK,MAAM;gBAAE,OAAO,CAAC,aAAa;YAC7C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE;gBAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC9E,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAC1B,GAAG,CAAC,UAAU,CAAC,iCAAiC,IAAI,wCAAwC,CAAC,CAAC;YAC9F,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,kBAAkB,CAAC,MAAc,EAAE,KAAmB;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;gBAC7C,MAAM,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;YAC/C,CAAC;YACD,sEAAsE;YACtE,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,oBAAoB,CAAC,MAAwB,EAAE,KAAyB;QAC9E,MAAM,IAAI,GAAG,KAAK,IAAI,SAAS,CAAC;QAChC,IAAI,IAAI,KAAK,MAAM,CAAC,cAAc,EAAE,CAAC;YACnC,aAAa,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,2BAA2B;YACrF,OAAO;QACT,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,IAAI,CAAC,IAAI,4BAA4B,IAAI,8CAA8C,CACnH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,oBAAoB,MAAM,CAAC,IAAI,CAAC,IAAI,uBAAuB,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,aAAa,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,gFAAgF;IACxE,GAAG,CACT,IAAY,EACZ,MAAc,EACd,IAAY,EACZ,IAAa,EACb,OAAgC;QAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,8BAA8B,CAAC,WAAW,IAAI,oBAAoB,CAAC,CACxE,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7B,MAAM,CACJ,IAAI,8BAA8B,CAChC,kBAAkB,IAAI,gBAAgB,MAAM,IAAI,IAAI,GAAG,CACxD,CACF,CAAC;YACJ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,cAAc,CAAC,CAAC;YAClD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE;gBACxB,KAAK;gBACL,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CACjB,OAAO,CAAC;oBACN,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;oBACzB,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAA4B;iBACpD,CAAC;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAChG,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IACpE,UAAU,CAChB,IAAY,EACZ,IAAY,EACZ,OAA+B,EAC/B,SAAiC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,WAAW,IAAI,oBAAoB,CAA0B,CAAC;YACtF,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,sCAAsC;YACnE,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClC,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,WAAW,IAAI,8BAA8B,CACrB,CAAC;gBAC3B,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,IAAI,iBAAiB,CAAC,CAAC;YACxD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE;gBAC7B,KAAK;gBACL,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;oBAChC,IAAI,MAAM,GAAG,KAAK,CAAC;oBACnB,MAAM,KAAK,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBAClD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;wBAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACzC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;wBAClC,IAAI,MAAM;4BAAE,OAAO;wBACnB,MAAM,GAAG,IAAI,CAAC;wBACd,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC,CAAC,CAAC;oBACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;wBACtB,IAAI,MAAM;4BAAE,OAAO;wBACnB,MAAM,GAAG,IAAI,CAAC;wBACd,SAAS,CAAC,OAAO,EAAE,CAAC;oBACtB,CAAC,CAAC,CAAC;oBACH,OAAO,CAAC;wBACN,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;4BAChB,IAAI,CAAC,MAAM;gCAAE,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC3D,CAAC;wBACD,KAAK,EAAE,GAAG,EAAE;4BACV,IAAI,MAAM;gCAAE,OAAO;4BACnB,MAAM,GAAG,IAAI,CAAC;4BACd,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,CAAC;qBACF,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU,CAAC,MAAc,EAAE,KAAmB;QAC1D,MAAM,MAAM,GAAG,CAAC,OAAe,EAAQ,EAAE;YACvC,GAAG,CAAC,UAAU,CAAC,iCAAiC,OAAO,EAAE,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;YACjC,MAAM,CAAC,qCAAqC,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC9C,MAAM,CAAC,oCAAoC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,MAAM,CAAC,0CAA0C,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YACjE,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,oBAAoB,KAAK,CAAC,IAAI,qCAAqC,CAAC,CAAC;YACjF,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,KAAa,EAAE,QAAgB;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,cAAc,CAAC,MAAc;QACnC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9C,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,qBAAqB,EAAE,CAAC","sourcesContent":["import { randomBytes } from \"node:crypto\";\nimport { readFileSync } from \"node:fs\";\nimport type { Server, Socket } from \"node:net\";\nimport { hostname } from \"node:os\";\nimport { TLSSocket, createServer as createTlsServer } from \"node:tls\";\nimport * as log from \"../log.js\";\nimport type { GondolinGatewaySettings } from \"../types.js\";\nimport { gondolinFleet } from \"./gondolin-fleet.js\";\nimport { gondolinJoin } from \"./gondolin-join.js\";\nimport {\n GondolinRemoteConnection,\n GondolinWorkerUnreachableError,\n createSessionFrameParser,\n encodeSessionMessage,\n} from \"./gondolin-remote.js\";\nimport type { SessionClient, SessionClientCallbacks } from \"./gondolin-worker-client.js\";\n\nconst RPC_TIMEOUT_MS = 150_000;\nconst TUNNEL_TIMEOUT_MS = 15_000;\n\n/** What a worker announces when it dials home (mirrors the Go register frame). */\ninterface GondolinWorkerRegistration {\n name: string;\n os?: string;\n arch?: string;\n accelerator?: string;\n cpus?: number;\n memoryBytes?: number;\n maxRuntimes?: number;\n protocolVersion?: number;\n runtimes?: Array<{ sessionId: string; instanceId: string }>;\n /** Non-empty when the worker's shared workspace failed its usability probe. */\n workspaceError?: string;\n}\n\ninterface ControlFrame extends GondolinWorkerRegistration {\n type: string;\n id?: number;\n status?: number;\n body?: unknown;\n nonce?: string;\n activeRuntimes?: number;\n /** join exchange */\n token?: string;\n csr?: string;\n}\n\ninterface RegisteredWorker {\n info: GondolinWorkerRegistration;\n socket: Socket;\n connectedAt: number;\n lastSeen: number;\n activeRuntimes: number;\n workspaceError?: string;\n pendingRpc: Map<number, { resolve: (frame: ControlFrame) => void; timer: NodeJS.Timeout }>;\n nextRpcId: number;\n}\n\n/**\n * Splits a byte stream into u32be-length-prefixed JSON frames, preserving any\n * bytes after the point where the consumer stops parsing (a tunnel connection\n * switches protocols after its preamble frame).\n */\nfunction createControlFrameReader(onFrame: (frame: ControlFrame, leftover: Buffer) => void): {\n push: (chunk: Buffer) => void;\n stop: () => void;\n} {\n let buffer = Buffer.alloc(0);\n let stopped = false;\n return {\n push(chunk: Buffer) {\n if (stopped) return;\n buffer = Buffer.concat([buffer, chunk]);\n for (;;) {\n // onFrame may call stop() synchronously (a tunnel switching protocols)\n if (stopped || buffer.length < 4) return;\n const length = buffer.readUInt32BE(0);\n if (buffer.length < 4 + length) return;\n const payload = buffer.subarray(4, 4 + length);\n buffer = buffer.subarray(4 + length);\n let frame: ControlFrame;\n try {\n frame = JSON.parse(payload.toString(\"utf8\")) as ControlFrame;\n } catch {\n continue; // skip malformed frames\n }\n onFrame(frame, buffer);\n }\n },\n stop() {\n stopped = true;\n },\n };\n}\n\nfunction encodeControlFrame(frame: object): Buffer {\n return encodeSessionMessage(frame);\n}\n\ninterface GatewayOverrides {\n /** Server factory (tests substitute a plain net.Server). */\n createServer?: (onConnection: (socket: Socket) => void) => Server;\n /** mTLS verification result for a socket (tests substitute outcomes). */\n isAuthorized?: (socket: Socket) => boolean;\n leaseTtlSeconds?: number;\n rpcTimeoutMs?: number;\n tunnelTimeoutMs?: number;\n}\n\nfunction defaultIsAuthorized(socket: Socket): boolean {\n return socket instanceof TLSSocket && socket.authorized;\n}\n\n/**\n * The host side of dial-home workers: one mTLS listener that accepts worker\n * control channels (register/ping/RPC responses) and per-command dial-back\n * tunnels, keeps the registry of connected workers, and attaches each one to\n * the fleet as a regular worker connection. Placement, leases, fencing, and\n * failover are unchanged — this only reverses who dials whom.\n */\nclass GondolinWorkerGateway {\n private settings?: GondolinGatewaySettings;\n private overrides: GatewayOverrides = {};\n private server?: Server;\n private readonly workers = new Map<string, RegisteredWorker>();\n private readonly pendingTunnels = new Map<\n string,\n { resolve: (value: { socket: Socket; leftover: Buffer }) => void; timer: NodeJS.Timeout }\n >();\n\n configure(settings?: GondolinGatewaySettings, overrides?: GatewayOverrides): void {\n this.stop();\n this.settings = settings;\n this.overrides = overrides ?? {};\n }\n\n isConfigured(): boolean {\n return this.settings !== undefined;\n }\n\n /** Registered workers and their liveness (admin/observability surface). */\n list(): Array<{\n name: string;\n connected: boolean;\n activeRuntimes: number;\n workspaceError?: string;\n info: GondolinWorkerRegistration;\n }> {\n return Array.from(this.workers.values()).map((worker) => ({\n name: worker.info.name,\n connected: !worker.socket.destroyed,\n activeRuntimes: worker.activeRuntimes,\n workspaceError: worker.workspaceError,\n info: worker.info,\n }));\n }\n\n /**\n * Start the listener. Without explicit certificate settings the gateway\n * provisions its own: a private worker CA plus a server certificate signed\n * by it (required for join's CA pinning to also authenticate the host).\n */\n async start(): Promise<void> {\n if (!this.settings || this.server) return;\n const settings = this.settings;\n if (!settings.workspaceRoot) {\n // dial-home workers inherit workspaceRoot from here; without it every\n // runtime is rejected with \"escapes the workspace root\"\n throw new Error(\n \"gondolin gateway requires workspaceRoot (the worker-side shared workspace path); \" +\n \"set sandbox.gondolin.remote.gateway.workspaceRoot\",\n );\n }\n let material: { certFile: string; keyFile: string; clientCaFile: string };\n if (settings.certFile && settings.keyFile && settings.clientCaFile) {\n material = {\n certFile: settings.certFile,\n keyFile: settings.keyFile,\n clientCaFile: settings.clientCaFile,\n };\n } else {\n if (!gondolinJoin.isConfigured()) {\n throw new Error(\"gondolin gateway auto-init requires the join service to be configured\");\n }\n const issued = await gondolinJoin.ensureServerCert(settings.hostnames ?? [hostname()]);\n material = { ...issued, clientCaFile: gondolinJoin.paths().caFile };\n }\n const factory =\n this.overrides.createServer ??\n ((onConnection: (socket: Socket) => void) =>\n createTlsServer(\n {\n cert: readFileSync(material.certFile),\n key: readFileSync(material.keyFile),\n ca: readFileSync(material.clientCaFile),\n requestCert: true,\n // unauthenticated connections may only send a join frame; the\n // per-frame authorization gate enforces that below\n rejectUnauthorized: false,\n minVersion: \"TLSv1.3\",\n },\n onConnection,\n ));\n this.server = factory((socket) => this.handleConnection(socket));\n this.server.listen(this.settings.port, () => {\n log.logInfo(`Gondolin worker gateway listening on :${this.settings?.port}`);\n });\n }\n\n stop(): void {\n for (const worker of this.workers.values()) {\n worker.socket.destroy();\n for (const pending of worker.pendingRpc.values()) clearTimeout(pending.timer);\n }\n this.workers.clear();\n for (const pending of this.pendingTunnels.values()) clearTimeout(pending.timer);\n this.pendingTunnels.clear();\n this.server?.close();\n this.server = undefined;\n }\n\n /** Routes one inbound connection by its first frame: control or tunnel. */\n private handleConnection(socket: Socket): void {\n socket.setNoDelay(true);\n let routed = false;\n const authorized = (this.overrides.isAuthorized ?? defaultIsAuthorized)(socket);\n const reader = createControlFrameReader((frame, leftover) => {\n if (!routed) {\n routed = true;\n if (frame.type === \"join\") {\n void this.handleJoin(socket, frame);\n } else if (!authorized) {\n // everything except join requires a CA-signed client certificate\n socket.destroy();\n } else if (frame.type === \"register\" && frame.name) {\n this.acceptControlChannel(socket, frame, reader);\n } else if (frame.type === \"tunnel\" && frame.nonce) {\n reader.stop();\n socket.removeAllListeners(\"data\");\n this.acceptTunnel(socket, frame.nonce, leftover);\n } else {\n socket.destroy();\n }\n return;\n }\n this.handleControlFrame(socket, frame);\n });\n socket.on(\"data\", (chunk: Buffer) => reader.push(chunk));\n socket.on(\"error\", () => socket.destroy());\n }\n\n private acceptControlChannel(\n socket: Socket,\n registration: ControlFrame,\n _reader: { stop: () => void },\n ): void {\n const name = registration.name as string;\n const previous = this.workers.get(name);\n if (previous && !previous.socket.destroyed) {\n log.logInfo(`Gondolin worker '${name}' re-registered; superseding previous connection`);\n previous.socket.destroy();\n }\n const worker: RegisteredWorker = {\n info: registration,\n socket,\n connectedAt: Date.now(),\n lastSeen: Date.now(),\n activeRuntimes: registration.runtimes?.length ?? 0,\n pendingRpc: new Map(),\n nextRpcId: 1,\n };\n this.workers.set(name, worker);\n const surviving = registration.runtimes?.length ?? 0;\n log.logInfo(\n `Gondolin worker joined: ${name} (${registration.os}/${registration.arch}, ` +\n `${registration.accelerator}, ${registration.cpus} cpus, ` +\n `cap ${registration.maxRuntimes ?? \"∞\"}, ${surviving} runtimes surviving)`,\n );\n\n const overrides = this.settings?.workers?.[name];\n gondolinFleet.attachWorker(\n {\n name,\n url: `dialhome://${name}`,\n workspaceRoot: this.settings?.workspaceRoot,\n maxRuntimes: overrides?.maxRuntimes ?? registration.maxRuntimes,\n draining: overrides?.draining,\n },\n (connectionOverrides) =>\n new GondolinRemoteConnection(\n { name, url: `dialhome://${name}`, workspaceRoot: this.settings?.workspaceRoot },\n {\n ...connectionOverrides,\n ...(this.overrides.leaseTtlSeconds !== undefined\n ? { leaseTtlSeconds: this.overrides.leaseTtlSeconds }\n : {}),\n request: (method, path, body, headers) => this.rpc(name, method, path, body, headers),\n tunnel: (path, headers, callbacks) => this.openTunnel(name, path, headers, callbacks),\n },\n ),\n );\n this.applyWorkspaceHealth(worker, registration.workspaceError);\n // a worker that reconnected may carry runtimes whose placements moved\n void gondolinFleet.reconcile();\n\n socket.on(\"close\", () => {\n const current = this.workers.get(name);\n if (current !== worker) return; // superseded\n for (const pending of worker.pendingRpc.values()) clearTimeout(pending.timer);\n worker.pendingRpc.clear();\n log.logWarning(`Gondolin worker disconnected: ${name} (placements fence until lease expiry)`);\n gondolinFleet.detachWorker(name);\n this.workers.delete(name);\n });\n }\n\n private handleControlFrame(socket: Socket, frame: ControlFrame): void {\n const worker = this.workerBySocket(socket);\n if (!worker) return;\n worker.lastSeen = Date.now();\n if (frame.type === \"ping\") {\n if (typeof frame.activeRuntimes === \"number\") {\n worker.activeRuntimes = frame.activeRuntimes;\n }\n // omitted on healthy pings (Go omitempty), so absence means recovered\n this.applyWorkspaceHealth(worker, frame.workspaceError);\n socket.write(encodeControlFrame({ type: \"pong\" }));\n return;\n }\n if (frame.type === \"response\" && typeof frame.id === \"number\") {\n const pending = worker.pendingRpc.get(frame.id);\n if (!pending) return;\n worker.pendingRpc.delete(frame.id);\n clearTimeout(pending.timer);\n pending.resolve(frame);\n }\n }\n\n /**\n * Track shared-workspace health from register/ping frames and push it into\n * the fleet, logging only the transitions. A worker whose NFS mount died\n * keeps its control channel alive — this is what turns that silent state\n * into refused placements and fail-fast execs.\n */\n private applyWorkspaceHealth(worker: RegisteredWorker, error: string | undefined): void {\n const next = error || undefined;\n if (next === worker.workspaceError) {\n gondolinFleet.setWorkspaceHealth(worker.info.name, next); // attach may have reset it\n return;\n }\n if (next) {\n log.logWarning(\n `Gondolin worker '${worker.info.name}' workspace is unusable: ${next} — new placements disabled until it recovers`,\n );\n } else {\n log.logInfo(`Gondolin worker '${worker.info.name}' workspace recovered`);\n }\n worker.workspaceError = next;\n gondolinFleet.setWorkspaceHealth(worker.info.name, next);\n }\n\n /** RPC to a worker over its control channel (the reverse of HTTPS requests). */\n private rpc(\n name: string,\n method: string,\n path: string,\n body?: object,\n headers?: Record<string, string>,\n ): Promise<{ status: number; json: Record<string, unknown> }> {\n const worker = this.workers.get(name);\n if (!worker || worker.socket.destroyed) {\n return Promise.reject(\n new GondolinWorkerUnreachableError(`worker '${name}' is not connected`),\n );\n }\n const id = worker.nextRpcId++;\n return new Promise((resolve, reject) => {\n const timer = setTimeout(() => {\n worker.pendingRpc.delete(id);\n reject(\n new GondolinWorkerUnreachableError(\n `RPC to worker '${name}' timed out (${method} ${path})`,\n ),\n );\n }, this.overrides.rpcTimeoutMs ?? RPC_TIMEOUT_MS);\n timer.unref?.();\n worker.pendingRpc.set(id, {\n timer,\n resolve: (frame) =>\n resolve({\n status: frame.status ?? 0,\n json: (frame.body ?? {}) as Record<string, unknown>,\n }),\n });\n worker.socket.write(encodeControlFrame({ type: \"request\", id, method, path, body, headers }));\n });\n }\n\n /** Ask the worker to dial back a data connection for one session tunnel. */\n private openTunnel(\n name: string,\n path: string,\n headers: Record<string, string>,\n callbacks: SessionClientCallbacks,\n ): Promise<SessionClient> {\n const worker = this.workers.get(name);\n if (!worker || worker.socket.destroyed) {\n const error = new Error(`worker '${name}' is not connected`) as NodeJS.ErrnoException;\n error.code = \"ECONNREFUSED\"; // nothing executed — safe to recreate\n return Promise.reject(error);\n }\n const sessionId = path.split(\"/\")[3];\n const nonce = randomBytes(16).toString(\"hex\");\n return new Promise((resolve, reject) => {\n const timer = setTimeout(() => {\n this.pendingTunnels.delete(nonce);\n const error = new Error(\n `worker '${name}' did not dial back a tunnel`,\n ) as NodeJS.ErrnoException;\n error.code = \"ECONNREFUSED\";\n reject(error);\n }, this.overrides.tunnelTimeoutMs ?? TUNNEL_TIMEOUT_MS);\n timer.unref?.();\n this.pendingTunnels.set(nonce, {\n timer,\n resolve: ({ socket, leftover }) => {\n let closed = false;\n const parse = createSessionFrameParser(callbacks);\n if (leftover.length > 0) parse(leftover);\n socket.on(\"data\", parse);\n socket.on(\"error\", (error: Error) => {\n if (closed) return;\n closed = true;\n callbacks.onClose(error);\n });\n socket.on(\"close\", () => {\n if (closed) return;\n closed = true;\n callbacks.onClose();\n });\n resolve({\n send: (message) => {\n if (!closed) socket.write(encodeSessionMessage(message));\n },\n close: () => {\n if (closed) return;\n closed = true;\n socket.destroy();\n },\n });\n },\n });\n worker.socket.write(encodeControlFrame({ type: \"open-tunnel\", nonce, sessionId, headers }));\n });\n }\n\n /**\n * One-time token → CA-signed client certificate. The only operation an\n * unauthenticated connection may perform; the connection closes after the\n * response either way.\n */\n private async handleJoin(socket: Socket, frame: ControlFrame): Promise<void> {\n const refuse = (message: string): void => {\n log.logWarning(`Gondolin worker join refused: ${message}`);\n socket.end(encodeControlFrame({ type: \"join-error\", message }));\n };\n if (!gondolinJoin.isConfigured()) {\n refuse(\"join is not enabled on this gateway\");\n return;\n }\n if (!frame.token || !frame.csr || !frame.name) {\n refuse(\"join requires token, csr, and name\");\n return;\n }\n if (!gondolinJoin.consumeToken(frame.token)) {\n refuse(`invalid or expired join token (worker '${frame.name}')`);\n return;\n }\n try {\n const signed = await gondolinJoin.signCsr(frame.csr);\n log.logInfo(`Gondolin worker '${frame.name}' joined: client certificate issued`);\n socket.end(encodeControlFrame({ type: \"join-ok\", cert: signed.certPem, ca: signed.caPem }));\n } catch (err) {\n refuse(`certificate issuance failed: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n\n private acceptTunnel(socket: Socket, nonce: string, leftover: Buffer): void {\n const pending = this.pendingTunnels.get(nonce);\n if (!pending) {\n socket.destroy();\n return;\n }\n this.pendingTunnels.delete(nonce);\n clearTimeout(pending.timer);\n pending.resolve({ socket, leftover: Buffer.from(leftover) });\n }\n\n private workerBySocket(socket: Socket): RegisteredWorker | undefined {\n for (const worker of this.workers.values()) {\n if (worker.socket === socket) return worker;\n }\n return undefined;\n }\n}\n\nexport const gondolinGateway = new GondolinWorkerGateway();\n"]}
@@ -0,0 +1,15 @@
1
+ interface NfsAdvisoryDeps {
2
+ os?: NodeJS.Platform;
3
+ run?: (cmd: string, args: string[]) => string;
4
+ emit?: (line: string) => void;
5
+ }
6
+ /**
7
+ * Startup advisory for gondolin:remote workers: they need the mikan-host
8
+ * workspace on shared storage, and the simplest way is to NFS-export it from
9
+ * the host itself. mikan does NOT touch root or system files — it only checks
10
+ * whether the workspace is already exported and, if not, prints the
11
+ * OS-appropriate commands the operator runs once by hand.
12
+ */
13
+ export declare function checkWorkspaceExported(workspaceDir: string, gatewayHostname?: string, deps?: NfsAdvisoryDeps): void;
14
+ export {};
15
+ //# sourceMappingURL=gondolin-nfs-advisory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gondolin-nfs-advisory.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-nfs-advisory.ts"],"names":[],"mappings":"AAIA,UAAU,eAAe;IACvB,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;IACrB,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IAC9C,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,EACpB,eAAe,CAAC,EAAE,MAAM,EACxB,IAAI,GAAE,eAAoB,GACzB,IAAI,CAcN"}
@@ -0,0 +1,60 @@
1
+ import { execFileSync } from "node:child_process";
2
+ import { platform } from "node:os";
3
+ import * as log from "../log.js";
4
+ /**
5
+ * Startup advisory for gondolin:remote workers: they need the mikan-host
6
+ * workspace on shared storage, and the simplest way is to NFS-export it from
7
+ * the host itself. mikan does NOT touch root or system files — it only checks
8
+ * whether the workspace is already exported and, if not, prints the
9
+ * OS-appropriate commands the operator runs once by hand.
10
+ */
11
+ export function checkWorkspaceExported(workspaceDir, gatewayHostname, deps = {}) {
12
+ const os = deps.os ?? platform();
13
+ const emit = deps.emit ?? ((line) => console.log(line));
14
+ if (os !== "darwin" && os !== "linux") {
15
+ log.logWarning(`gondolin:remote workers need ${workspaceDir} on shared storage; NFS auto-detection is unavailable on ${os}`);
16
+ return;
17
+ }
18
+ if (isExported(os, workspaceDir, deps.run)) {
19
+ log.logInfo(`Workspace ${workspaceDir} is NFS-exported for remote workers`);
20
+ return;
21
+ }
22
+ for (const line of setupLines(os, workspaceDir, gatewayHostname))
23
+ emit(line);
24
+ }
25
+ function isExported(os, workspaceDir, run) {
26
+ const exec = run ?? ((cmd, args) => execFileSync(cmd, args, { encoding: "utf8" }));
27
+ try {
28
+ if (os === "darwin") {
29
+ const out = exec("showmount", ["-e", "localhost"]);
30
+ return out.split("\n").some((line) => line.trimStart().startsWith(workspaceDir));
31
+ }
32
+ // linux: exportfs -s lists current exports
33
+ const out = exec("exportfs", ["-s"]);
34
+ return out
35
+ .split("\n")
36
+ .some((line) => line.startsWith(workspaceDir + " ") || line === workspaceDir);
37
+ }
38
+ catch {
39
+ // no NFS server running / tool missing → treat as not exported
40
+ return false;
41
+ }
42
+ }
43
+ function setupLines(os, workspaceDir, gatewayHostname) {
44
+ const hostHint = gatewayHostname ?? "<host-tailscale-ip>";
45
+ const lines = [
46
+ "",
47
+ " ⚠ gondolin:remote workers need this workspace on shared storage.",
48
+ ` Export it from this host once (mikan will not modify system files):`,
49
+ "",
50
+ ];
51
+ if (os === "darwin") {
52
+ lines.push(` echo '${workspaceDir} -network 100.64.0.0 -mask 255.192.0.0 -mapall=$(id -u):$(id -g)' | sudo tee -a /etc/exports`, ` sudo nfsd enable && sudo nfsd start`);
53
+ }
54
+ else {
55
+ lines.push(` echo '${workspaceDir} 100.64.0.0/10(rw,sync,no_subtree_check,all_squash,anonuid=$(id -u),anongid=$(id -g))' | sudo tee -a /etc/exports`, ` sudo exportfs -ra && sudo systemctl enable --now nfs-server`);
56
+ }
57
+ lines.push("", " Then on each worker (over tailscale), mount it and point --workspace-root at it:", ` sudo mount -t nfs -o vers=3,resvport,nolock ${hostHint}:${workspaceDir} /mnt/mikan-workspace`, "", " 100.64.0.0/10 is the tailscale range; -mapall/all_squash makes worker", " writes owned by you on the host. Skip this only if you already share the", " workspace another way (existing NFS, GCS-fuse, etc.).", "");
58
+ return lines;
59
+ }
60
+ //# sourceMappingURL=gondolin-nfs-advisory.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gondolin-nfs-advisory.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-nfs-advisory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAQjC;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,YAAoB,EACpB,eAAwB,EACxB,IAAI,GAAoB,EAAE;IAE1B,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAChE,IAAI,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;QACtC,GAAG,CAAC,UAAU,CACZ,gCAAgC,YAAY,4DAA4D,EAAE,EAAE,CAC7G,CAAC;QACF,OAAO;IACT,CAAC;IACD,IAAI,UAAU,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,OAAO,CAAC,aAAa,YAAY,qCAAqC,CAAC,CAAC;QAC5E,OAAO;IACT,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,EAAE,EAAE,YAAY,EAAE,eAAe,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/E,CAAC;AAED,SAAS,UAAU,CACjB,EAAmB,EACnB,YAAoB,EACpB,GAA2B;IAE3B,MAAM,IAAI,GACR,GAAG,IAAI,CAAC,CAAC,GAAW,EAAE,IAAc,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1F,IAAI,CAAC;QACH,IAAI,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;YACnD,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;QACnF,CAAC;QACD,2CAA2C;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACrC,OAAO,GAAG;aACP,KAAK,CAAC,IAAI,CAAC;aACX,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,GAAG,GAAG,CAAC,IAAI,IAAI,KAAK,YAAY,CAAC,CAAC;IAClF,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;QAC/D,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,EAAmB,EAAE,YAAoB,EAAE,eAAwB;IACrF,MAAM,QAAQ,GAAG,eAAe,IAAI,qBAAqB,CAAC;IAC1D,MAAM,KAAK,GAAa;QACtB,EAAE;QACF,oEAAoE;QACpE,yEAAyE;QACzE,EAAE;KACH,CAAC;IACF,IAAI,EAAE,KAAK,QAAQ,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CACR,aAAa,YAAY,8FAA8F,EACvH,yCAAyC,CAC1C,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CACR,aAAa,YAAY,mHAAmH,EAC5I,iEAAiE,CAClE,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CACR,EAAE,EACF,sFAAsF,EACtF,mDAAmD,QAAQ,IAAI,YAAY,uBAAuB,EAClG,EAAE,EACF,2EAA2E,EAC3E,8EAA8E,EAC9E,2DAA2D,EAC3D,EAAE,CACH,CAAC;IACF,OAAO,KAAK,CAAC;AACf,CAAC","sourcesContent":["import { execFileSync } from \"node:child_process\";\nimport { platform } from \"node:os\";\nimport * as log from \"../log.js\";\n\ninterface NfsAdvisoryDeps {\n os?: NodeJS.Platform;\n run?: (cmd: string, args: string[]) => string;\n emit?: (line: string) => void;\n}\n\n/**\n * Startup advisory for gondolin:remote workers: they need the mikan-host\n * workspace on shared storage, and the simplest way is to NFS-export it from\n * the host itself. mikan does NOT touch root or system files — it only checks\n * whether the workspace is already exported and, if not, prints the\n * OS-appropriate commands the operator runs once by hand.\n */\nexport function checkWorkspaceExported(\n workspaceDir: string,\n gatewayHostname?: string,\n deps: NfsAdvisoryDeps = {},\n): void {\n const os = deps.os ?? platform();\n const emit = deps.emit ?? ((line: string) => console.log(line));\n if (os !== \"darwin\" && os !== \"linux\") {\n log.logWarning(\n `gondolin:remote workers need ${workspaceDir} on shared storage; NFS auto-detection is unavailable on ${os}`,\n );\n return;\n }\n if (isExported(os, workspaceDir, deps.run)) {\n log.logInfo(`Workspace ${workspaceDir} is NFS-exported for remote workers`);\n return;\n }\n for (const line of setupLines(os, workspaceDir, gatewayHostname)) emit(line);\n}\n\nfunction isExported(\n os: NodeJS.Platform,\n workspaceDir: string,\n run: NfsAdvisoryDeps[\"run\"],\n): boolean {\n const exec =\n run ?? ((cmd: string, args: string[]) => execFileSync(cmd, args, { encoding: \"utf8\" }));\n try {\n if (os === \"darwin\") {\n const out = exec(\"showmount\", [\"-e\", \"localhost\"]);\n return out.split(\"\\n\").some((line) => line.trimStart().startsWith(workspaceDir));\n }\n // linux: exportfs -s lists current exports\n const out = exec(\"exportfs\", [\"-s\"]);\n return out\n .split(\"\\n\")\n .some((line) => line.startsWith(workspaceDir + \" \") || line === workspaceDir);\n } catch {\n // no NFS server running / tool missing → treat as not exported\n return false;\n }\n}\n\nfunction setupLines(os: NodeJS.Platform, workspaceDir: string, gatewayHostname?: string): string[] {\n const hostHint = gatewayHostname ?? \"<host-tailscale-ip>\";\n const lines: string[] = [\n \"\",\n \" ⚠ gondolin:remote workers need this workspace on shared storage.\",\n ` Export it from this host once (mikan will not modify system files):`,\n \"\",\n ];\n if (os === \"darwin\") {\n lines.push(\n ` echo '${workspaceDir} -network 100.64.0.0 -mask 255.192.0.0 -mapall=$(id -u):$(id -g)' | sudo tee -a /etc/exports`,\n ` sudo nfsd enable && sudo nfsd start`,\n );\n } else {\n lines.push(\n ` echo '${workspaceDir} 100.64.0.0/10(rw,sync,no_subtree_check,all_squash,anonuid=$(id -u),anongid=$(id -g))' | sudo tee -a /etc/exports`,\n ` sudo exportfs -ra && sudo systemctl enable --now nfs-server`,\n );\n }\n lines.push(\n \"\",\n \" Then on each worker (over tailscale), mount it and point --workspace-root at it:\",\n ` sudo mount -t nfs -o vers=3,resvport,nolock ${hostHint}:${workspaceDir} /mnt/mikan-workspace`,\n \"\",\n \" 100.64.0.0/10 is the tailscale range; -mapall/all_squash makes worker\",\n \" writes owned by you on the host. Skip this only if you already share the\",\n \" workspace another way (existing NFS, GCS-fuse, etc.).\",\n \"\",\n );\n return lines;\n}\n"]}
@@ -49,6 +49,15 @@ export declare class GondolinRemoteConnection {
49
49
  env?: Record<string, string>;
50
50
  signal?: AbortSignal;
51
51
  }): Promise<ExecResult>;
52
+ /**
53
+ * Split mounts for the remote worker:
54
+ * - sources under the mikan-host workspace are translated to the worker's
55
+ * shared `workspaceRoot` (shared POSIX storage), and
56
+ * - sources outside it that are files (vault credentials, which live in the
57
+ * host state dir, not the shared workspace) are read and shipped as
58
+ * content payloads for the worker to project into the guest.
59
+ * A non-file source outside the workspace has nowhere to go and is dropped.
60
+ */
52
61
  private translateMounts;
53
62
  private leaseFor;
54
63
  private renew;
@@ -1 +1 @@
1
- {"version":3,"file":"gondolin-remote.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-remote.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAGL,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAC5B,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAa7C,UAAU,cAAc;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,6EAA6E;AAC7E,qBAAa,8BAA+B,SAAQ,KAAK;CAAG;AAE5D,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,CACR,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAC7B,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,CACP,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,SAAS,EAAE,sBAAsB,KAC9B,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;CACrE;AAED,qFAAqF;AACrF,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,sBAAsB,GAChC,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAqBzB;AAED,uEAAuE;AACvE,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAK5D;AAED;;;;;GAKG;AACH,qBAAa,wBAAwB;IAOjC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAP5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,GAAG,CAAC,CAA6C;IAEzD,YACmB,QAAQ,EAAE,4BAA4B,EACtC,SAAS,GAAE,uBAA4B,EAIzD;IAED,gEAAgE;IAChE,OAAO,IAAI,IAAI,CAGd;IAEK,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAQ/C;IAEK,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAoC1F;IAEK,IAAI,CACR,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,WAAW,GAAG,WAAW,GAAG,YAAY,CAAC,GAC5E,OAAO,CAAC,IAAI,CAAC,CAYf;IAEK,cAAc,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOpE;IAED,kEAAkE;IAC5D,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAO9E;IAEK,IAAI,CACR,MAAM,EAAE,qBAAqB,EAC7B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GACnE,OAAO,CAAC,UAAU,CAAC,CAiBrB;IAED,OAAO,CAAC,eAAe;YAiBT,QAAQ;YAuBR,KAAK;IAiBnB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,OAAO;IAmDf,OAAO,CAAC,UAAU;IA8DlB,OAAO,CAAC,UAAU;CAcnB"}
1
+ {"version":3,"file":"gondolin-remote.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-remote.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAGL,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAC5B,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAa7C,UAAU,cAAc;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,6EAA6E;AAC7E,qBAAa,8BAA+B,SAAQ,KAAK;CAAG;AAE5D,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,CACR,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAC7B,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7B,MAAM,CAAC,EAAE,CACP,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,SAAS,EAAE,sBAAsB,KAC9B,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iFAAiF;IACjF,eAAe,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;CACrE;AAED,qFAAqF;AACrF,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,sBAAsB,GAChC,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAqBzB;AAED,uEAAuE;AACvE,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAK5D;AAED;;;;;GAKG;AACH,qBAAa,wBAAwB;IAOjC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAP5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,GAAG,CAAC,CAA6C;IAEzD,YACmB,QAAQ,EAAE,4BAA4B,EACtC,SAAS,GAAE,uBAA4B,EAIzD;IAED,gEAAgE;IAChE,OAAO,IAAI,IAAI,CAGd;IAEK,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAQ/C;IAEK,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAsC1F;IAEK,IAAI,CACR,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,WAAW,GAAG,WAAW,GAAG,YAAY,CAAC,GAC5E,OAAO,CAAC,IAAI,CAAC,CAYf;IAEK,cAAc,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOpE;IAED,kEAAkE;IAC5D,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAO9E;IAEK,IAAI,CACR,MAAM,EAAE,qBAAqB,EAC7B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GACnE,OAAO,CAAC,UAAU,CAAC,CAiBrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,eAAe;YAkCT,QAAQ;YAuBR,KAAK;IAiBnB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,OAAO;IAmDf,OAAO,CAAC,UAAU;IA8DlB,OAAO,CAAC,UAAU;CAcnB"}
@@ -1,4 +1,4 @@
1
- import { readFileSync } from "node:fs";
1
+ import { readFileSync, statSync } from "node:fs";
2
2
  import { request as httpsRequest } from "node:https";
3
3
  import { relative, isAbsolute, join } from "node:path";
4
4
  import * as log from "../log.js";
@@ -71,10 +71,12 @@ export class GondolinRemoteConnection {
71
71
  return response.json;
72
72
  }
73
73
  async ensure(instanceId, spec) {
74
+ const { mounts, credentialFiles } = this.translateMounts(spec);
74
75
  const body = {
75
76
  instanceId,
76
77
  imageSelector: spec.image,
77
- mounts: this.translateMounts(spec),
78
+ mounts,
79
+ credentialFiles,
78
80
  cpus: spec.cpus ?? "",
79
81
  memory: spec.memory ?? "",
80
82
  fingerprint: spec.fingerprint,
@@ -134,20 +136,43 @@ export class GondolinRemoteConnection {
134
136
  }
135
137
  return execOverSessionConnect((callbacks) => this.openTunnel(handle.instanceId, `/v1/runtimes/${handle.sessionId}/session`, this.leaseHeaders(lease), callbacks), command, options);
136
138
  }
139
+ /**
140
+ * Split mounts for the remote worker:
141
+ * - sources under the mikan-host workspace are translated to the worker's
142
+ * shared `workspaceRoot` (shared POSIX storage), and
143
+ * - sources outside it that are files (vault credentials, which live in the
144
+ * host state dir, not the shared workspace) are read and shipped as
145
+ * content payloads for the worker to project into the guest.
146
+ * A non-file source outside the workspace has nowhere to go and is dropped.
147
+ */
137
148
  translateMounts(spec) {
138
149
  const root = this.settings.workspaceRoot;
139
- if (!root || !spec.workspacePath)
140
- return spec.mounts;
141
- const translated = [];
150
+ if (!root || !spec.workspacePath) {
151
+ return { mounts: spec.mounts, credentialFiles: [] };
152
+ }
153
+ const mounts = [];
154
+ const credentialFiles = [];
142
155
  for (const mount of spec.mounts) {
143
156
  const suffix = relative(spec.workspacePath, mount.source);
144
- if (suffix.startsWith("..") || isAbsolute(suffix)) {
145
- log.logWarning(`Skipping mount outside the shared workspace for remote runtime: ${mount.source}`);
157
+ if (!suffix.startsWith("..") && !isAbsolute(suffix)) {
158
+ mounts.push({ source: join(root, suffix), target: mount.target });
146
159
  continue;
147
160
  }
148
- translated.push({ source: join(root, suffix), target: mount.target });
161
+ try {
162
+ if (!statSync(mount.source).isFile()) {
163
+ log.logWarning(`Skipping directory mount outside the shared workspace for remote runtime: ${mount.source}`);
164
+ continue;
165
+ }
166
+ credentialFiles.push({
167
+ target: mount.target,
168
+ contentBase64: readFileSync(mount.source).toString("base64"),
169
+ });
170
+ }
171
+ catch {
172
+ // missing source: skip, matching the local worker's behavior
173
+ }
149
174
  }
150
- return translated;
175
+ return { mounts, credentialFiles };
151
176
  }
152
177
  async leaseFor(instanceId) {
153
178
  const cached = this.leases.get(instanceId);