@tagma/completion-llm-judge 0.2.73 → 0.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -41,7 +41,7 @@ pipeline:
|
|
|
41
41
|
include the test name, the file path, and the assertion that
|
|
42
42
|
failed. The output must not be an empty placeholder.
|
|
43
43
|
# endpoint / model / api_key_env all default to local Ollama + qwen3:4b
|
|
44
|
-
timeout:
|
|
44
|
+
timeout: 2h
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
Swap to a hosted backend:
|
|
@@ -72,7 +72,7 @@ await tagma.registry.loadPlugins(['@tagma/completion-llm-judge'], process.cwd())
|
|
|
72
72
|
| `model` | string | `qwen3:4b` | Judge model name. Swap for `qwen3:8b`, `deepseek-r1:7b`, `gpt-4o-mini`, etc. |
|
|
73
73
|
| `endpoint` | string | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible chat completions URL. Default points at local Ollama |
|
|
74
74
|
| `api_key_env` | string | _(none)_ | Env var holding the bearer token. Leave unset for local Ollama; set for hosted backends |
|
|
75
|
-
| `timeout` | duration | `
|
|
75
|
+
| `timeout` | duration | `2h` | Max time to wait for the judge response (reasoning models need more time than chat models) |
|
|
76
76
|
| `max_output_chars` | number | `8000` | Truncate task stdout before judging (head+tail preserved) |
|
|
77
77
|
|
|
78
78
|
## Behavior
|
package/dist/index.js
CHANGED
|
@@ -35,7 +35,7 @@ const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
|
|
|
35
35
|
// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
|
|
36
36
|
// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
|
|
37
37
|
const DEFAULT_MODEL = 'qwen3:4b';
|
|
38
|
-
const DEFAULT_TIMEOUT_MS =
|
|
38
|
+
const DEFAULT_TIMEOUT_MS = 2 * 60 * 60 * 1_000;
|
|
39
39
|
const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
|
|
40
40
|
const SYSTEM_PROMPT = 'You are a strict quality judge for task outputs.\n' +
|
|
41
41
|
'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
|
|
@@ -177,7 +177,7 @@ export const LlmJudgeCompletion = {
|
|
|
177
177
|
},
|
|
178
178
|
timeout: {
|
|
179
179
|
type: 'duration',
|
|
180
|
-
default: '
|
|
180
|
+
default: '2h',
|
|
181
181
|
description: 'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
|
|
182
182
|
},
|
|
183
183
|
max_output_chars: {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,0BAA0B,GAM3B,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,0BAA0B,GAM3B,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AAC/C,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,yBAAyB,CAAC,QAAa,EAAE,SAA6B;IAC7E,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,+CAA+C,QAAQ,CAAC,MAAM,GAAG,CACjH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,sBAAsB,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,uEAAuE,MAAM,CAAC,QAAQ,GAAG,CAC1F,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAErF,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,WAAW,GAAG,mBAAmB,CACpC,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAC5D,CAAC;QACF,yBAAyB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACjF,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,KAAK,MAAM,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;gBACF,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,QAAQ,EACN,qBAAqB,SAAS,IAAI,SAAS,IAAI;wBAC/C,gBAAgB,CAAC,OAAO,EAAE,wBAAwB,CAAC;iBACtD,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,6BAA6B,GAAG,EAAE,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,YAAY,EAAE;QACZ,WAAW,EAAE;YACX,SAAS,EAAE,kBAAkB;SAC9B;KACF;CACoB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tagma/completion-llm-judge",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.74",
|
|
4
4
|
"description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"prepublishOnly": "bun run build"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@tagma/types": "0.4.
|
|
53
|
+
"@tagma/types": "0.4.86"
|
|
54
54
|
},
|
|
55
55
|
"peerDependencies": {
|
|
56
56
|
"@tagma/types": ">=0.4.18 <0.5.0"
|