@tagma/completion-llm-judge 0.2.6 → 0.2.7

package/dist/index.js

@@ -202,7 +202,7 @@ export const LlmJudgeCompletion = {
         try {
             const content = await callJudge(endpoint, model, apiKey, messages, timeoutMs, ctx.signal);
             const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
-            const passed = firstLine.startsWith('PASS');
+            const passed = firstLine === 'PASS';
             if (!passed) {
                 // Surface the judge's reasoning in logs so pipeline operators can
                 // see why an output was rejected without re-running it themselves.

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,iBAAiB,GAKlB,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,YAAY,EAAE;QACZ,WAAW,EAAE;YACX,SAAS,EAAE,kBAAkB;SAC9B;KACF;CACoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,iBAAiB,GAKlB,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,KAAK,MAAM,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,YAAY,EAAE;QACZ,WAAW,EAAE;YACX,SAAS,EAAE,kBAAkB;SAC9B;KACF;CACoB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tagma/completion-llm-judge",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
   "license": "MIT",
   "repository": {
@@ -23,8 +23,7 @@
     "./package.json": "./package.json"
   },
   "files": [
-    "dist",
-    "src"
+    "dist"
   ],
   "publishConfig": {
     "access": "public"
@@ -45,11 +44,11 @@
     "type": "llm_judge"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json",
+    "build": "node ../../scripts/build-package.mjs",
     "test": "bun test",
     "prepublishOnly": "bun run build"
   },
-  "peerDependencies": {
-    "@tagma/types": "0.4.11"
+  "dependencies": {
+    "@tagma/types": "0.4.13"
   }
 }

package/src/index.test.ts

@@ -1,16 +0,0 @@
-import { describe, expect, test } from 'bun:test';
-import plugin, { LlmJudgeCompletion } from './index';
-import manifest from '../package.json' with { type: 'json' };
-
-describe('completion-llm-judge plugin shape', () => {
-  test('default export is a capability plugin matching package manifest', () => {
-    expect(manifest.tagmaPlugin.category).toBe('completions');
-    expect(manifest.tagmaPlugin.type).toBe('llm_judge');
-    expect(plugin.name).toBe(manifest.name);
-    expect(plugin.capabilities?.completions?.[manifest.tagmaPlugin.type]).toBe(LlmJudgeCompletion);
-  });
-
-  test('check is a function', () => {
-    expect(typeof plugin.capabilities!.completions!.llm_judge.check).toBe('function');
-  });
-});

package/src/index.ts

@@ -1,279 +0,0 @@
-// ═══ LLM-as-Judge Completion Plugin ═══
-//
-// Uses an OpenAI-compatible chat completions endpoint to verify whether a
-// task's output satisfies a rubric. Complements the deterministic built-in
-// completions (`exit_code`, `file_exists`, `output_check`) with AI-powered
-// checks — useful when success is defined semantically rather than by a
-// grep-able pattern or a file on disk.
-//
-// Default backend is a **local Ollama** server using its OpenAI-compatible
-// route (`/v1/chat/completions`, available since Ollama 0.1.29+), with
-// `qwen3:4b` as a small, cheap-to-run reasoning model. No API key is
-// required for local Ollama; remote endpoints can set `api_key_env` to
-// whatever header the server expects.
-//
-// The judge is instructed to answer PASS/FAIL on the first line. Reasoning
-// models (qwen3, deepseek-r1, etc.) emit `<think>...</think>` blocks in
-// their message content — we strip those before parsing, so the rubric
-// works the same whether the judge model is a thinker or not.
-//
-// Usage in pipeline.yaml:
-//   plugins: ["@tagma/completion-llm-judge"]
-//   tracks:
-//     - tasks:
-//         - id: draft
-//           completion:
-//             type: llm_judge
-//             rubric: "Output must list at least 3 failing tests with file paths."
-//             # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
-
-import {
-  parseDurationSafe,
-  type TagmaPlugin,
-  type CompletionPlugin,
-  type CompletionContext,
-  type TaskResult,
-} from '@tagma/types';
-
-// Ollama exposes an OpenAI-compatible `/v1/chat/completions` route on port
-// 11434 by default. Point this at any OpenAI-compatible server (OpenAI,
-// vLLM, llama.cpp, LM Studio, Groq, Together, etc.) to swap backends.
-const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
-// qwen3:4b is a small reasoning model (~2.5 GB on disk, runs on CPU) that
-// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
-// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
-const DEFAULT_MODEL = 'qwen3:4b';
-const DEFAULT_TIMEOUT_MS = 120_000;
-const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
-
-const SYSTEM_PROMPT =
-  'You are a strict quality judge for task outputs.\n' +
-  'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
-  'On subsequent lines you may provide a one-sentence justification.\n' +
-  'Do not use any other format. Do not wrap the answer in code fences.\n' +
-  'If you reason step-by-step internally, still put PASS or FAIL on the first line of your final answer.';
-
-interface ChatMessage {
-  readonly role: 'system' | 'user';
-  readonly content: string;
-}
-
-interface ChatCompletionResponse {
-  readonly choices?: ReadonlyArray<{
-    readonly message?: { readonly content?: string };
-  }>;
-}
-
-// Head-and-tail truncation preserves the start of the output (where agents
-// usually declare intent) and the end (where they summarize results),
-// dropping the middle when the combined length exceeds the budget. This
-// keeps the judge's view of the output meaningful even for very long runs.
-function truncateForJudge(text: string, maxChars: number): string {
-  if (text.length <= maxChars) return text;
-  const marker = '\n...[truncated]...\n';
-  const budget = maxChars - marker.length;
-  if (budget <= 0) return text.slice(0, maxChars);
-  const head = Math.floor(budget * 0.7);
-  const tail = budget - head;
-  return text.slice(0, head) + marker + text.slice(-tail);
-}
-
-// Strip reasoning-model thinking blocks so verdict parsing sees the real
-// answer. Qwen3 and DeepSeek-R1 both emit `<think>...</think>` inline in
-// message content when served via Ollama's OpenAI-compat route. We also
-// drop the legacy `<thinking>` variant some fine-tunes use. Applied
-// before any trimming so leading whitespace from the stripped block
-// doesn't leak into the first line.
-function stripThinking(content: string): string {
-  return content
-    .replace(/<think>[\s\S]*?<\/think>/gi, '')
-    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
-    .trim();
-}
-
-async function callJudge(
-  endpoint: string,
-  model: string,
-  apiKey: string | undefined,
-  messages: readonly ChatMessage[],
-  timeoutMs: number,
-  externalSignal: AbortSignal | undefined,
-): Promise<string> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeoutMs);
-
-  const onExternalAbort = (): void => controller.abort();
-  if (externalSignal) {
-    if (externalSignal.aborted) {
-      controller.abort();
-    } else {
-      externalSignal.addEventListener('abort', onExternalAbort, { once: true });
-    }
-  }
-
-  try {
-    const headers: Record<string, string> = {
-      'content-type': 'application/json',
-    };
-    // Only send Authorization when we actually have a key — local Ollama
-    // doesn't require one, and some OpenAI-compat proxies reject bogus
-    // placeholder tokens like "ollama" with 401 instead of ignoring them.
-    if (apiKey) {
-      headers.authorization = `Bearer ${apiKey}`;
-    }
-
-    const res = await fetch(endpoint, {
-      method: 'POST',
-      headers,
-      body: JSON.stringify({
-        model,
-        messages,
-        temperature: 0,
-        // `stream: false` is already the default but we set it explicitly
-        // because Ollama's OpenAI-compat route streams by default in some
-        // older versions.
-        stream: false,
-      }),
-      signal: controller.signal,
-    });
-    if (!res.ok) {
-      const text = await res.text().catch(() => '');
-      throw new Error(`judge endpoint ${res.status}: ${text.slice(0, 200)}`);
-    }
-    const payload = (await res.json()) as ChatCompletionResponse;
-    const content = payload.choices?.[0]?.message?.content;
-    if (typeof content !== 'string') {
-      throw new Error('judge endpoint returned no message content');
-    }
-    return stripThinking(content);
-  } finally {
-    clearTimeout(timer);
-    if (externalSignal) externalSignal.removeEventListener('abort', onExternalAbort);
-  }
-}
-
-export const LlmJudgeCompletion: CompletionPlugin = {
-  name: 'llm_judge',
-  schema: {
-    description:
-      'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
-    fields: {
-      rubric: {
-        type: 'string',
-        required: true,
-        description: 'Criteria the judge should verify. Plain language.',
-        placeholder: 'Output must list at least 3 failing tests with file paths.',
-      },
-      model: {
-        type: 'string',
-        default: DEFAULT_MODEL,
-        description:
-          'Judge model name. Default is a small Ollama reasoning model (qwen3:4b). Swap for qwen3:8b, deepseek-r1:7b, or a hosted model for stricter judging.',
-        placeholder: DEFAULT_MODEL,
-      },
-      endpoint: {
-        type: 'string',
-        default: DEFAULT_ENDPOINT,
-        description:
-          'OpenAI-compatible chat completions endpoint. Defaults to local Ollama (http://localhost:11434/v1/chat/completions).',
-        placeholder: DEFAULT_ENDPOINT,
-      },
-      api_key_env: {
-        type: 'string',
-        description:
-          'Env var containing the bearer token for the judge endpoint. Leave unset for local Ollama; set to OPENAI_API_KEY etc. for hosted backends.',
-        placeholder: 'OPENAI_API_KEY',
-      },
-      timeout: {
-        type: 'duration',
-        default: '120s',
-        description:
-          'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
-      },
-      max_output_chars: {
-        type: 'number',
-        default: DEFAULT_MAX_OUTPUT_CHARS,
-        min: 500,
-        max: 200_000,
-        description: 'Truncate task stdout to this many chars before judging.',
-      },
-    },
-  },
-
-  async check(
-    config: Record<string, unknown>,
-    result: TaskResult,
-    ctx: CompletionContext,
-  ): Promise<boolean> {
-    const rubric = config.rubric as string | undefined;
-    if (!rubric) throw new Error('llm_judge completion: "rubric" is required');
-
-    // api_key_env is optional — when unset we talk to the endpoint
-    // anonymously (correct for local Ollama). When the user names an env
-    // var, we require it to be populated so config errors fail loudly
-    // instead of silently stripping auth.
-    const apiKeyEnv = config.api_key_env as string | undefined;
-    let apiKey: string | undefined;
-    if (apiKeyEnv) {
-      apiKey = process.env[apiKeyEnv];
-      if (!apiKey) {
-        throw new Error(`llm_judge completion: env var ${apiKeyEnv} is not set`);
-      }
-    }
-
-    const model = (config.model as string | undefined) ?? DEFAULT_MODEL;
-    const endpoint = (config.endpoint as string | undefined) ?? DEFAULT_ENDPOINT;
-    const timeoutMs = parseDurationSafe(config.timeout, DEFAULT_TIMEOUT_MS);
-    const maxChars =
-      typeof config.max_output_chars === 'number' && config.max_output_chars > 0
-        ? Math.floor(config.max_output_chars)
-        : DEFAULT_MAX_OUTPUT_CHARS;
-
-    // Prefer the driver-normalized text (e.g. concatenated message text
-    // from AI drivers that emit NDJSON). Feeding raw NDJSON to the judge
-    // wastes tokens and obscures the semantic output the judge is meant
-    // to grade. Command tasks and drivers without parseResult fall back
-    // to raw stdout, which for them IS the semantic output.
-    const taskOutput = result.normalizedOutput ?? result.stdout;
-
-    const userContent =
-      `[Rubric]\n${rubric}\n\n` +
-      `[Exit Code]\n${result.exitCode}\n\n` +
-      `[Task Output]\n${truncateForJudge(taskOutput, maxChars)}`;
-
-    const messages: ChatMessage[] = [
-      { role: 'system', content: SYSTEM_PROMPT },
-      { role: 'user', content: userContent },
-    ];
-
-    try {
-      const content = await callJudge(endpoint, model, apiKey, messages, timeoutMs, ctx.signal);
-      const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
-      const passed = firstLine.startsWith('PASS');
-      if (!passed) {
-        // Surface the judge's reasoning in logs so pipeline operators can
-        // see why an output was rejected without re-running it themselves.
-        console.warn(
-          `[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`,
-        );
-      }
-      return passed;
-    } catch (err) {
-      // Treat judge failures as FAIL: a completion gate that errors open
-      // is worse than one that errors closed. Operators can re-run the
-      // task once the judge endpoint is healthy again.
-      const msg = err instanceof Error ? err.message : String(err);
-      console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
-      return false;
-    }
-  },
-};
-
-export default {
-  name: '@tagma/completion-llm-judge',
-  capabilities: {
-    completions: {
-      llm_judge: LlmJudgeCompletion,
-    },
-  },
-} satisfies TagmaPlugin;