@tagma/completion-llm-judge 0.2.41 → 0.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -78,6 +78,7 @@ await tagma.registry.loadPlugins(['@tagma/completion-llm-judge'], process.cwd())
|
|
|
78
78
|
## Behavior
|
|
79
79
|
|
|
80
80
|
- **Verdict format**: the judge is instructed to answer `PASS` or `FAIL` on the first line. Missing or ambiguous answers are treated as FAIL — a judge that errs open defeats the purpose of the gate.
|
|
81
|
+
- **Repair feedback**: a FAIL verdict returns the bounded judge explanation as structured completion feedback, so a workflow with `lifecycle.repair: true` can act on the unmet rubric in its next attempt.
|
|
81
82
|
- **Reasoning-model support**: `<think>...</think>` and `<thinking>...</thinking>` blocks are stripped from the response before verdict parsing, so qwen3, DeepSeek-R1, and other thinkers work without any extra config.
|
|
82
83
|
- **Truncation**: task stdout longer than `max_output_chars` is truncated head-and-tail (70/30 split) with a marker in the middle, so the judge still sees the task's intent and its final summary.
|
|
83
84
|
- **Error handling**: network errors, auth errors, timeout, or malformed responses all mark the task as not-complete and log a warning with the judge's verbatim response (if any).
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,EAIL,KAAK,gBAAgB,EAGtB,MAAM,cAAc,CAAC;AAoJtB,eAAO,MAAM,kBAAkB,EAAE,gBA6HhC,CAAC;;;;;;;;;AAEF,wBAOwB"}
|
package/dist/index.js
CHANGED
|
@@ -234,8 +234,13 @@ export const LlmJudgeCompletion = {
|
|
|
234
234
|
// Surface the judge's reasoning in logs so pipeline operators can
|
|
235
235
|
// see why an output was rejected without re-running it themselves.
|
|
236
236
|
console.warn(`[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`);
|
|
237
|
+
return {
|
|
238
|
+
passed: false,
|
|
239
|
+
feedback: `llm_judge verdict=${firstLine || '<empty>'}\n` +
|
|
240
|
+
truncateForJudge(content, DEFAULT_MAX_OUTPUT_CHARS),
|
|
241
|
+
};
|
|
237
242
|
}
|
|
238
|
-
return
|
|
243
|
+
return true;
|
|
239
244
|
}
|
|
240
245
|
catch (err) {
|
|
241
246
|
// Treat judge failures as FAIL: a completion gate that errors open
|
|
@@ -243,7 +248,7 @@ export const LlmJudgeCompletion = {
|
|
|
243
248
|
// task once the judge endpoint is healthy again.
|
|
244
249
|
const msg = err instanceof Error ? err.message : String(err);
|
|
245
250
|
console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
|
|
246
|
-
return false;
|
|
251
|
+
return { passed: false, feedback: `llm_judge request failed: ${msg}` };
|
|
247
252
|
}
|
|
248
253
|
},
|
|
249
254
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,0BAA0B,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,0BAA0B,GAM3B,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,yBAAyB,CAAC,QAAa,EAAE,SAA6B;IAC7E,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,+CAA+C,QAAQ,CAAC,MAAM,GAAG,CACjH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,sBAAsB,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,uEAAuE,MAAM,CAAC,QAAQ,GAAG,CAC1F,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAErF,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,WAAW,GAAG,mBAAmB,CACpC,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAC5D,CAAC;QACF,yBAAyB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACjF,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,KAAK,MAAM,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;gBACF,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,QAAQ,EACN,qBAAqB,SAAS,IAAI,SAAS,IAAI;wBAC/C,gBAAgB,CAAC,OAAO,EAAE,wBAAwB,CAAC;iBACtD,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,6BAA6B,GAAG,EAAE,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,YAAY,EAAE;QACZ,WAAW,EAAE;YACX,SAAS,EAAE,kBAAkB;SAC9B;KACF;CACoB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tagma/completion-llm-judge",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.43",
|
|
4
4
|
"description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"prepublishOnly": "bun run build"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@tagma/types": "0.4.
|
|
53
|
+
"@tagma/types": "0.4.55"
|
|
54
54
|
},
|
|
55
55
|
"peerDependencies": {
|
|
56
56
|
"@tagma/types": ">=0.4.18 <0.5.0"
|