@tagma/completion-llm-judge 0.1.3 → 0.2.0

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,gBAAgB,EAAiC,MAAM,cAAc,CAAC;AA0IpF,QAAA,MAAM,kBAAkB,EAAE,gBA4GzB,CAAC;AAGF,eAAO,MAAM,cAAc,gBAAgB,CAAC;AAC5C,eAAO,MAAM,UAAU,cAAc,CAAC;AACtC,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,gBAAgB,EAAiC,MAAM,cAAc,CAAC;AA0IpF,QAAA,MAAM,kBAAkB,EAAE,gBAmHzB,CAAC;AAGF,eAAO,MAAM,cAAc,gBAAgB,CAAC;AAC5C,eAAO,MAAM,UAAU,cAAc,CAAC;AACtC,eAAe,kBAAkB,CAAC"}

package/dist/index.js

@@ -205,9 +205,15 @@ const LlmJudgeCompletion = {
         const maxChars = typeof config.max_output_chars === 'number' && config.max_output_chars > 0
             ? Math.floor(config.max_output_chars)
             : DEFAULT_MAX_OUTPUT_CHARS;
+        // Prefer the driver-normalized text (e.g. concatenated message text
+        // from AI drivers that emit NDJSON). Feeding raw NDJSON to the judge
+        // wastes tokens and obscures the semantic output the judge is meant
+        // to grade. Command tasks and drivers without parseResult fall back
+        // to raw stdout, which for them IS the semantic output.
+        const taskOutput = result.normalizedOutput ?? result.stdout;
         const userContent = `[Rubric]\n${rubric}\n\n` +
             `[Exit Code]\n${result.exitCode}\n\n` +
-            `[Task Output]\n${truncateForJudge(result.stdout, maxChars)}`;
+            `[Task Output]\n${truncateForJudge(taskOutput, maxChars)}`;
         const messages = [
             { role: 'system', content: SYSTEM_PROMPT },
             { role: 'user', content: userContent },

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAItF,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,SAAS,iBAAiB,CAAC,GAAY,EAAE,QAAgB;IACvD,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACpD,IAAI,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACxB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,KAAK,IAAI;YACP,OAAO,CAAC,CAAC;QACX,KAAK,GAAG;YACN,OAAO,CAAC,GAAG,MAAM,CAAC;QACpB,KAAK,GAAG;YACN,OAAO,CAAC,GAAG,SAAS,CAAC;QACvB,KAAK,GAAG,CAAC;QACT;YACE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpB,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,kBAAkB,GAAqB;IAC3C,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;QAEhE,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,0CAA0C;AAC1C,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC;AAC5C,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;AACtC,eAAe,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAItF,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,SAAS,iBAAiB,CAAC,GAAY,EAAE,QAAgB;IACvD,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACpD,IAAI,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACxB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,KAAK,IAAI;YACP,OAAO,CAAC,CAAC;QACX,KAAK,GAAG;YACN,OAAO,CAAC,GAAG,MAAM,CAAC;QACpB,KAAK,GAAG;YACN,OAAO,CAAC,GAAG,SAAS,CAAC;QACvB,KAAK,GAAG,CAAC;QACT;YACE,OAAO,CAAC,GAAG,IAAI,CAAC;IACpB,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,kBAAkB,GAAqB;IAC3C,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,0CAA0C;AAC1C,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC;AAC5C,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;AACtC,eAAe,kBAAkB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tagma/completion-llm-judge",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
   "license": "MIT",
   "repository": {
@@ -49,7 +49,7 @@
     "prepublishOnly": "bun run build"
   },
   "peerDependencies": {
-    "@tagma/types": "0.2.7"
+    "@tagma/types": "0.4.0"
   },
   "devDependencies": {
     "bun-types": "^1.3.11",

package/src/index.ts

@@ -1,281 +1,288 @@
-// ═══ LLM-as-Judge Completion Plugin ═══
-//
-// Uses an OpenAI-compatible chat completions endpoint to verify whether a
-// task's output satisfies a rubric. Complements the deterministic built-in
-// completions (`exit_code`, `file_exists`, `output_check`) with AI-powered
-// checks — useful when success is defined semantically rather than by a
-// grep-able pattern or a file on disk.
-//
-// Default backend is a **local Ollama** server using its OpenAI-compatible
-// route (`/v1/chat/completions`, available since Ollama 0.1.29+), with
-// `qwen3:4b` as a small, cheap-to-run reasoning model. No API key is
-// required for local Ollama; remote endpoints can set `api_key_env` to
-// whatever header the server expects.
-//
-// The judge is instructed to answer PASS/FAIL on the first line. Reasoning
-// models (qwen3, deepseek-r1, etc.) emit `<think>...</think>` blocks in
-// their message content — we strip those before parsing, so the rubric
-// works the same whether the judge model is a thinker or not.
-//
-// Usage in pipeline.yaml:
-//   plugins: ["@tagma/completion-llm-judge"]
-//   tracks:
-//     - tasks:
-//         - id: draft
-//           completion:
-//             type: llm_judge
-//             rubric: "Output must list at least 3 failing tests with file paths."
-//             # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
-
-import type { CompletionPlugin, CompletionContext, TaskResult } from '@tagma/types';
-
-// Ollama exposes an OpenAI-compatible `/v1/chat/completions` route on port
-// 11434 by default. Point this at any OpenAI-compatible server (OpenAI,
-// vLLM, llama.cpp, LM Studio, Groq, Together, etc.) to swap backends.
-const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
-// qwen3:4b is a small reasoning model (~2.5 GB on disk, runs on CPU) that
-// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
-// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
-const DEFAULT_MODEL = 'qwen3:4b';
-const DEFAULT_TIMEOUT_MS = 120_000;
-const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
-
-const SYSTEM_PROMPT =
-  'You are a strict quality judge for task outputs.\n' +
-  'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
-  'On subsequent lines you may provide a one-sentence justification.\n' +
-  'Do not use any other format. Do not wrap the answer in code fences.\n' +
-  'If you reason step-by-step internally, still put PASS or FAIL on the first line of your final answer.';
-
-interface ChatMessage {
-  readonly role: 'system' | 'user';
-  readonly content: string;
-}
-
-interface ChatCompletionResponse {
-  readonly choices?: ReadonlyArray<{
-    readonly message?: { readonly content?: string };
-  }>;
-}
-
-function parseDurationSafe(raw: unknown, fallback: number): number {
-  if (raw == null) return fallback;
-  const str = String(raw).trim();
-  const m = str.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
-  if (!m) return fallback;
-  const n = Number(m[1]);
-  switch (m[2]) {
-    case 'ms':
-      return n;
-    case 'm':
-      return n * 60_000;
-    case 'h':
-      return n * 3_600_000;
-    case 's':
-    default:
-      return n * 1000;
-  }
-}
-
-// Head-and-tail truncation preserves the start of the output (where agents
-// usually declare intent) and the end (where they summarize results),
-// dropping the middle when the combined length exceeds the budget. This
-// keeps the judge's view of the output meaningful even for very long runs.
-function truncateForJudge(text: string, maxChars: number): string {
-  if (text.length <= maxChars) return text;
-  const marker = '\n...[truncated]...\n';
-  const budget = maxChars - marker.length;
-  if (budget <= 0) return text.slice(0, maxChars);
-  const head = Math.floor(budget * 0.7);
-  const tail = budget - head;
-  return text.slice(0, head) + marker + text.slice(-tail);
-}
-
-// Strip reasoning-model thinking blocks so verdict parsing sees the real
-// answer. Qwen3 and DeepSeek-R1 both emit `<think>...</think>` inline in
-// message content when served via Ollama's OpenAI-compat route. We also
-// drop the legacy `<thinking>` variant some fine-tunes use. Applied
-// before any trimming so leading whitespace from the stripped block
-// doesn't leak into the first line.
-function stripThinking(content: string): string {
-  return content
-    .replace(/<think>[\s\S]*?<\/think>/gi, '')
-    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
-    .trim();
-}
-
-async function callJudge(
-  endpoint: string,
-  model: string,
-  apiKey: string | undefined,
-  messages: readonly ChatMessage[],
-  timeoutMs: number,
-  externalSignal: AbortSignal | undefined,
-): Promise<string> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeoutMs);
-
-  const onExternalAbort = (): void => controller.abort();
-  if (externalSignal) {
-    if (externalSignal.aborted) {
-      controller.abort();
-    } else {
-      externalSignal.addEventListener('abort', onExternalAbort, { once: true });
-    }
-  }
-
-  try {
-    const headers: Record<string, string> = {
-      'content-type': 'application/json',
-    };
-    // Only send Authorization when we actually have a key — local Ollama
-    // doesn't require one, and some OpenAI-compat proxies reject bogus
-    // placeholder tokens like "ollama" with 401 instead of ignoring them.
-    if (apiKey) {
-      headers.authorization = `Bearer ${apiKey}`;
-    }
-
-    const res = await fetch(endpoint, {
-      method: 'POST',
-      headers,
-      body: JSON.stringify({
-        model,
-        messages,
-        temperature: 0,
-        // `stream: false` is already the default but we set it explicitly
-        // because Ollama's OpenAI-compat route streams by default in some
-        // older versions.
-        stream: false,
-      }),
-      signal: controller.signal,
-    });
-    if (!res.ok) {
-      const text = await res.text().catch(() => '');
-      throw new Error(`judge endpoint ${res.status}: ${text.slice(0, 200)}`);
-    }
-    const payload = (await res.json()) as ChatCompletionResponse;
-    const content = payload.choices?.[0]?.message?.content;
-    if (typeof content !== 'string') {
-      throw new Error('judge endpoint returned no message content');
-    }
-    return stripThinking(content);
-  } finally {
-    clearTimeout(timer);
-    if (externalSignal) externalSignal.removeEventListener('abort', onExternalAbort);
-  }
-}
-
-const LlmJudgeCompletion: CompletionPlugin = {
-  name: 'llm_judge',
-  schema: {
-    description:
-      'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
-    fields: {
-      rubric: {
-        type: 'string',
-        required: true,
-        description: 'Criteria the judge should verify. Plain language.',
-        placeholder: 'Output must list at least 3 failing tests with file paths.',
-      },
-      model: {
-        type: 'string',
-        default: DEFAULT_MODEL,
-        description:
-          'Judge model name. Default is a small Ollama reasoning model (qwen3:4b). Swap for qwen3:8b, deepseek-r1:7b, or a hosted model for stricter judging.',
-        placeholder: DEFAULT_MODEL,
-      },
-      endpoint: {
-        type: 'string',
-        default: DEFAULT_ENDPOINT,
-        description:
-          'OpenAI-compatible chat completions endpoint. Defaults to local Ollama (http://localhost:11434/v1/chat/completions).',
-        placeholder: DEFAULT_ENDPOINT,
-      },
-      api_key_env: {
-        type: 'string',
-        description:
-          'Env var containing the bearer token for the judge endpoint. Leave unset for local Ollama; set to OPENAI_API_KEY etc. for hosted backends.',
-        placeholder: 'OPENAI_API_KEY',
-      },
-      timeout: {
-        type: 'duration',
-        default: '120s',
-        description:
-          'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
-      },
-      max_output_chars: {
-        type: 'number',
-        default: DEFAULT_MAX_OUTPUT_CHARS,
-        min: 500,
-        max: 200_000,
-        description: 'Truncate task stdout to this many chars before judging.',
-      },
-    },
-  },
-
-  async check(
-    config: Record<string, unknown>,
-    result: TaskResult,
-    ctx: CompletionContext,
-  ): Promise<boolean> {
-    const rubric = config.rubric as string | undefined;
-    if (!rubric) throw new Error('llm_judge completion: "rubric" is required');
-
-    // api_key_env is optional — when unset we talk to the endpoint
-    // anonymously (correct for local Ollama). When the user names an env
-    // var, we require it to be populated so config errors fail loudly
-    // instead of silently stripping auth.
-    const apiKeyEnv = config.api_key_env as string | undefined;
-    let apiKey: string | undefined;
-    if (apiKeyEnv) {
-      apiKey = process.env[apiKeyEnv];
-      if (!apiKey) {
-        throw new Error(`llm_judge completion: env var ${apiKeyEnv} is not set`);
-      }
-    }
-
-    const model = (config.model as string | undefined) ?? DEFAULT_MODEL;
-    const endpoint = (config.endpoint as string | undefined) ?? DEFAULT_ENDPOINT;
-    const timeoutMs = parseDurationSafe(config.timeout, DEFAULT_TIMEOUT_MS);
-    const maxChars =
-      typeof config.max_output_chars === 'number' && config.max_output_chars > 0
-        ? Math.floor(config.max_output_chars)
-        : DEFAULT_MAX_OUTPUT_CHARS;
-
-    const userContent =
-      `[Rubric]\n${rubric}\n\n` +
-      `[Exit Code]\n${result.exitCode}\n\n` +
-      `[Task Output]\n${truncateForJudge(result.stdout, maxChars)}`;
-
-    const messages: ChatMessage[] = [
-      { role: 'system', content: SYSTEM_PROMPT },
-      { role: 'user', content: userContent },
-    ];
-
-    try {
-      const content = await callJudge(endpoint, model, apiKey, messages, timeoutMs, ctx.signal);
-      const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
-      const passed = firstLine.startsWith('PASS');
-      if (!passed) {
-        // Surface the judge's reasoning in logs so pipeline operators can
-        // see why an output was rejected without re-running it themselves.
-        console.warn(
-          `[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`,
-        );
-      }
-      return passed;
-    } catch (err) {
-      // Treat judge failures as FAIL: a completion gate that errors open
-      // is worse than one that errors closed. Operators can re-run the
-      // task once the judge endpoint is healthy again.
-      const msg = err instanceof Error ? err.message : String(err);
-      console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
-      return false;
-    }
-  },
-};
-
-// ═══ Plugin self-description exports ═══
-export const pluginCategory = 'completions';
-export const pluginType = 'llm_judge';
-export default LlmJudgeCompletion;
+// ═══ LLM-as-Judge Completion Plugin ═══
+//
+// Uses an OpenAI-compatible chat completions endpoint to verify whether a
+// task's output satisfies a rubric. Complements the deterministic built-in
+// completions (`exit_code`, `file_exists`, `output_check`) with AI-powered
+// checks — useful when success is defined semantically rather than by a
+// grep-able pattern or a file on disk.
+//
+// Default backend is a **local Ollama** server using its OpenAI-compatible
+// route (`/v1/chat/completions`, available since Ollama 0.1.29+), with
+// `qwen3:4b` as a small, cheap-to-run reasoning model. No API key is
+// required for local Ollama; remote endpoints can set `api_key_env` to
+// whatever header the server expects.
+//
+// The judge is instructed to answer PASS/FAIL on the first line. Reasoning
+// models (qwen3, deepseek-r1, etc.) emit `<think>...</think>` blocks in
+// their message content — we strip those before parsing, so the rubric
+// works the same whether the judge model is a thinker or not.
+//
+// Usage in pipeline.yaml:
+//   plugins: ["@tagma/completion-llm-judge"]
+//   tracks:
+//     - tasks:
+//         - id: draft
+//           completion:
+//             type: llm_judge
+//             rubric: "Output must list at least 3 failing tests with file paths."
+//             # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
+
+import type { CompletionPlugin, CompletionContext, TaskResult } from '@tagma/types';
+
+// Ollama exposes an OpenAI-compatible `/v1/chat/completions` route on port
+// 11434 by default. Point this at any OpenAI-compatible server (OpenAI,
+// vLLM, llama.cpp, LM Studio, Groq, Together, etc.) to swap backends.
+const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
+// qwen3:4b is a small reasoning model (~2.5 GB on disk, runs on CPU) that
+// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
+// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
+const DEFAULT_MODEL = 'qwen3:4b';
+const DEFAULT_TIMEOUT_MS = 120_000;
+const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
+
+const SYSTEM_PROMPT =
+  'You are a strict quality judge for task outputs.\n' +
+  'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
+  'On subsequent lines you may provide a one-sentence justification.\n' +
+  'Do not use any other format. Do not wrap the answer in code fences.\n' +
+  'If you reason step-by-step internally, still put PASS or FAIL on the first line of your final answer.';
+
+interface ChatMessage {
+  readonly role: 'system' | 'user';
+  readonly content: string;
+}
+
+interface ChatCompletionResponse {
+  readonly choices?: ReadonlyArray<{
+    readonly message?: { readonly content?: string };
+  }>;
+}
+
+function parseDurationSafe(raw: unknown, fallback: number): number {
+  if (raw == null) return fallback;
+  const str = String(raw).trim();
+  const m = str.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
+  if (!m) return fallback;
+  const n = Number(m[1]);
+  switch (m[2]) {
+    case 'ms':
+      return n;
+    case 'm':
+      return n * 60_000;
+    case 'h':
+      return n * 3_600_000;
+    case 's':
+    default:
+      return n * 1000;
+  }
+}
+
+// Head-and-tail truncation preserves the start of the output (where agents
+// usually declare intent) and the end (where they summarize results),
+// dropping the middle when the combined length exceeds the budget. This
+// keeps the judge's view of the output meaningful even for very long runs.
+function truncateForJudge(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  const marker = '\n...[truncated]...\n';
+  const budget = maxChars - marker.length;
+  if (budget <= 0) return text.slice(0, maxChars);
+  const head = Math.floor(budget * 0.7);
+  const tail = budget - head;
+  return text.slice(0, head) + marker + text.slice(-tail);
+}
+
+// Strip reasoning-model thinking blocks so verdict parsing sees the real
+// answer. Qwen3 and DeepSeek-R1 both emit `<think>...</think>` inline in
+// message content when served via Ollama's OpenAI-compat route. We also
+// drop the legacy `<thinking>` variant some fine-tunes use. Applied
+// before any trimming so leading whitespace from the stripped block
+// doesn't leak into the first line.
+function stripThinking(content: string): string {
+  return content
+    .replace(/<think>[\s\S]*?<\/think>/gi, '')
+    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+    .trim();
+}
+
+async function callJudge(
+  endpoint: string,
+  model: string,
+  apiKey: string | undefined,
+  messages: readonly ChatMessage[],
+  timeoutMs: number,
+  externalSignal: AbortSignal | undefined,
+): Promise<string> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  const onExternalAbort = (): void => controller.abort();
+  if (externalSignal) {
+    if (externalSignal.aborted) {
+      controller.abort();
+    } else {
+      externalSignal.addEventListener('abort', onExternalAbort, { once: true });
+    }
+  }
+
+  try {
+    const headers: Record<string, string> = {
+      'content-type': 'application/json',
+    };
+    // Only send Authorization when we actually have a key — local Ollama
+    // doesn't require one, and some OpenAI-compat proxies reject bogus
+    // placeholder tokens like "ollama" with 401 instead of ignoring them.
+    if (apiKey) {
+      headers.authorization = `Bearer ${apiKey}`;
+    }
+
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: 0,
+        // `stream: false` is already the default but we set it explicitly
+        // because Ollama's OpenAI-compat route streams by default in some
+        // older versions.
+        stream: false,
+      }),
+      signal: controller.signal,
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      throw new Error(`judge endpoint ${res.status}: ${text.slice(0, 200)}`);
+    }
+    const payload = (await res.json()) as ChatCompletionResponse;
+    const content = payload.choices?.[0]?.message?.content;
+    if (typeof content !== 'string') {
+      throw new Error('judge endpoint returned no message content');
+    }
+    return stripThinking(content);
+  } finally {
+    clearTimeout(timer);
+    if (externalSignal) externalSignal.removeEventListener('abort', onExternalAbort);
+  }
+}
+
+const LlmJudgeCompletion: CompletionPlugin = {
+  name: 'llm_judge',
+  schema: {
+    description:
+      'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
+    fields: {
+      rubric: {
+        type: 'string',
+        required: true,
+        description: 'Criteria the judge should verify. Plain language.',
+        placeholder: 'Output must list at least 3 failing tests with file paths.',
+      },
+      model: {
+        type: 'string',
+        default: DEFAULT_MODEL,
+        description:
+          'Judge model name. Default is a small Ollama reasoning model (qwen3:4b). Swap for qwen3:8b, deepseek-r1:7b, or a hosted model for stricter judging.',
+        placeholder: DEFAULT_MODEL,
+      },
+      endpoint: {
+        type: 'string',
+        default: DEFAULT_ENDPOINT,
+        description:
+          'OpenAI-compatible chat completions endpoint. Defaults to local Ollama (http://localhost:11434/v1/chat/completions).',
+        placeholder: DEFAULT_ENDPOINT,
+      },
+      api_key_env: {
+        type: 'string',
+        description:
+          'Env var containing the bearer token for the judge endpoint. Leave unset for local Ollama; set to OPENAI_API_KEY etc. for hosted backends.',
+        placeholder: 'OPENAI_API_KEY',
+      },
+      timeout: {
+        type: 'duration',
+        default: '120s',
+        description:
+          'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
+      },
+      max_output_chars: {
+        type: 'number',
+        default: DEFAULT_MAX_OUTPUT_CHARS,
+        min: 500,
+        max: 200_000,
+        description: 'Truncate task stdout to this many chars before judging.',
+      },
+    },
+  },
+
+  async check(
+    config: Record<string, unknown>,
+    result: TaskResult,
+    ctx: CompletionContext,
+  ): Promise<boolean> {
+    const rubric = config.rubric as string | undefined;
+    if (!rubric) throw new Error('llm_judge completion: "rubric" is required');
+
+    // api_key_env is optional — when unset we talk to the endpoint
+    // anonymously (correct for local Ollama). When the user names an env
+    // var, we require it to be populated so config errors fail loudly
+    // instead of silently stripping auth.
+    const apiKeyEnv = config.api_key_env as string | undefined;
+    let apiKey: string | undefined;
+    if (apiKeyEnv) {
+      apiKey = process.env[apiKeyEnv];
+      if (!apiKey) {
+        throw new Error(`llm_judge completion: env var ${apiKeyEnv} is not set`);
+      }
+    }
+
+    const model = (config.model as string | undefined) ?? DEFAULT_MODEL;
+    const endpoint = (config.endpoint as string | undefined) ?? DEFAULT_ENDPOINT;
+    const timeoutMs = parseDurationSafe(config.timeout, DEFAULT_TIMEOUT_MS);
+    const maxChars =
+      typeof config.max_output_chars === 'number' && config.max_output_chars > 0
+        ? Math.floor(config.max_output_chars)
+        : DEFAULT_MAX_OUTPUT_CHARS;
+
+    // Prefer the driver-normalized text (e.g. concatenated message text
+    // from AI drivers that emit NDJSON). Feeding raw NDJSON to the judge
+    // wastes tokens and obscures the semantic output the judge is meant
+    // to grade. Command tasks and drivers without parseResult fall back
+    // to raw stdout, which for them IS the semantic output.
+    const taskOutput = result.normalizedOutput ?? result.stdout;
+
+    const userContent =
+      `[Rubric]\n${rubric}\n\n` +
+      `[Exit Code]\n${result.exitCode}\n\n` +
+      `[Task Output]\n${truncateForJudge(taskOutput, maxChars)}`;
+
+    const messages: ChatMessage[] = [
+      { role: 'system', content: SYSTEM_PROMPT },
+      { role: 'user', content: userContent },
+    ];
+
+    try {
+      const content = await callJudge(endpoint, model, apiKey, messages, timeoutMs, ctx.signal);
+      const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
+      const passed = firstLine.startsWith('PASS');
+      if (!passed) {
+        // Surface the judge's reasoning in logs so pipeline operators can
+        // see why an output was rejected without re-running it themselves.
+        console.warn(
+          `[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`,
+        );
+      }
+      return passed;
+    } catch (err) {
+      // Treat judge failures as FAIL: a completion gate that errors open
+      // is worse than one that errors closed. Operators can re-run the
+      // task once the judge endpoint is healthy again.
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
+      return false;
+    }
+  },
+};
+
+// ═══ Plugin self-description exports ═══
+export const pluginCategory = 'completions';
+export const pluginType = 'llm_judge';
+export default LlmJudgeCompletion;