@tagma/completion-llm-judge 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tagma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,100 @@
+# @tagma/completion-llm-judge
+
+LLM-as-judge completion plugin for [@tagma/sdk](https://www.npmjs.com/package/@tagma/sdk).
+
+Uses an OpenAI-compatible chat completions endpoint to verify whether a task's output satisfies a rubric. Complements the deterministic built-in completions (`exit_code`, `file_exists`, `output_check`) when task success is defined semantically rather than by a grep-able pattern.
+
+**Default backend**: local [Ollama](https://ollama.com/) with `qwen3:4b` — a small reasoning model that runs on CPU with no API key. Swap `endpoint` + `model` to point at any OpenAI-compatible server (OpenAI, vLLM, llama.cpp, LM Studio, Groq, Together, OpenRouter, ...).
+
+## Install
+
+```bash
+bun add @tagma/completion-llm-judge
+```
+
+Then make sure Ollama is running with the default model pulled:
+
+```bash
+ollama pull qwen3:4b
+ollama serve  # usually auto-started
+```
+
+## Usage
+
+```yaml
+pipeline:
+  name: qa-loop
+  plugins:
+    - "@tagma/completion-llm-judge"
+  tracks:
+    - id: qa
+      name: QA
+      driver: claude-code
+      tasks:
+        - id: find-bugs
+          name: Find failing tests
+          prompt: "List all failing tests in the current workspace with their file paths."
+          output: ./output/find-bugs.md
+          completion:
+            type: llm_judge
+            rubric: |
+              The output must list at least 3 failing tests. Each entry must
+              include the test name, the file path, and the assertion that
+              failed. The output must not be an empty placeholder.
+            # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
+            timeout: 120s
+```
+
+Swap to a hosted backend:
+
+```yaml
+completion:
+  type: llm_judge
+  rubric: "..."
+  endpoint: https://api.openai.com/v1/chat/completions
+  model: gpt-4o-mini
+  api_key_env: OPENAI_API_KEY
+```
+
+Or load it programmatically:
+
+```ts
+import { bootstrapBuiltins, loadPlugins } from '@tagma/sdk';
+
+bootstrapBuiltins();
+await loadPlugins(['@tagma/completion-llm-judge']);
+```
+
+## Config
+
+| Field              | Type     | Default                                         | Notes                                                                                     |
+|--------------------|----------|-------------------------------------------------|-------------------------------------------------------------------------------------------|
+| `rubric`           | string   | *(required)*                                    | Plain-language success criteria the judge should verify                                   |
+| `model`            | string   | `qwen3:4b`                                      | Judge model name. Swap for `qwen3:8b`, `deepseek-r1:7b`, `gpt-4o-mini`, etc.              |
+| `endpoint`         | string   | `http://localhost:11434/v1/chat/completions`    | OpenAI-compatible chat completions URL. Default points at local Ollama                    |
+| `api_key_env`      | string   | *(none)*                                        | Env var holding the bearer token. Leave unset for local Ollama; set for hosted backends   |
+| `timeout`          | duration | `120s`                                          | Max time to wait for the judge response (reasoning models need more time than chat models)|
+| `max_output_chars` | number   | `8000`                                          | Truncate task stdout before judging (head+tail preserved)                                 |
+
+## Behavior
+
+- **Verdict format**: the judge is instructed to answer `PASS` or `FAIL` on the first line. Missing or ambiguous answers are treated as FAIL — a judge that errs open defeats the purpose of the gate.
+- **Reasoning-model support**: `<think>...</think>` and `<thinking>...</thinking>` blocks are stripped from the response before verdict parsing, so qwen3, DeepSeek-R1, and other thinkers work without any extra config.
+- **Truncation**: task stdout longer than `max_output_chars` is truncated head-and-tail (70/30 split) with a marker in the middle, so the judge still sees the task's intent and its final summary.
+- **Error handling**: network errors, auth errors, timeout, or malformed responses all mark the task as not-complete and log a warning with the judge's verbatim response (if any).
+- **Abort propagation**: the pipeline abort signal is wired into the judge fetch call, so cancelling a pipeline also cancels any in-flight judge request.
+- **Determinism**: the call uses `temperature: 0` to keep verdicts as stable as the model allows.
+
+## Alternative endpoints
+
+Any OpenAI-compatible endpoint works — just point `endpoint` and `api_key_env` at it:
+
+- **Local Ollama** (default): `http://localhost:11434/v1/chat/completions`, no API key
+- **OpenAI**: `https://api.openai.com/v1/chat/completions`, `api_key_env: OPENAI_API_KEY`
+- **Local models** via LM Studio, vLLM, llama.cpp OpenAI-compatible servers
+- **Hosted**: Groq, Together, Fireworks, OpenRouter, DeepInfra, etc.
+- **Anthropic** via an OpenAI-compat proxy (e.g. `anthropic-openai-proxy`)
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+import type { CompletionPlugin } from '@tagma/types';
+declare const LlmJudgeCompletion: CompletionPlugin;
+export declare const pluginCategory = "completions";
+export declare const pluginType = "llm_judge";
+export default LlmJudgeCompletion;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EACV,gBAAgB,EACjB,MAAM,cAAc,CAAC;AAsItB,QAAA,MAAM,kBAAkB,EAAE,gBA6GzB,CAAC;AAGF,eAAO,MAAM,cAAc,gBAAgB,CAAC;AAC5C,eAAO,MAAM,UAAU,cAAc,CAAC;AACtC,eAAe,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,236 @@
+// ═══ LLM-as-Judge Completion Plugin ═══
+//
+// Uses an OpenAI-compatible chat completions endpoint to verify whether a
+// task's output satisfies a rubric. Complements the deterministic built-in
+// completions (`exit_code`, `file_exists`, `output_check`) with AI-powered
+// checks — useful when success is defined semantically rather than by a
+// grep-able pattern or a file on disk.
+//
+// Default backend is a **local Ollama** server using its OpenAI-compatible
+// route (`/v1/chat/completions`, available since Ollama 0.1.29+), with
+// `qwen3:4b` as a small, cheap-to-run reasoning model. No API key is
+// required for local Ollama; remote endpoints can set `api_key_env` to
+// whatever header the server expects.
+//
+// The judge is instructed to answer PASS/FAIL on the first line. Reasoning
+// models (qwen3, deepseek-r1, etc.) emit `<think>...</think>` blocks in
+// their message content — we strip those before parsing, so the rubric
+// works the same whether the judge model is a thinker or not.
+//
+// Usage in pipeline.yaml:
+//   plugins: ["@tagma/completion-llm-judge"]
+//   tracks:
+//     - tasks:
+//         - id: draft
+//           completion:
+//             type: llm_judge
+//             rubric: "Output must list at least 3 failing tests with file paths."
+//             # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
+// Ollama exposes an OpenAI-compatible `/v1/chat/completions` route on port
+// 11434 by default. Point this at any OpenAI-compatible server (OpenAI,
+// vLLM, llama.cpp, LM Studio, Groq, Together, etc.) to swap backends.
+const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
+// qwen3:4b is a small reasoning model (~2.5 GB on disk, runs on CPU) that
+// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
+// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
+const DEFAULT_MODEL = 'qwen3:4b';
+const DEFAULT_TIMEOUT_MS = 120_000;
+const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
+const SYSTEM_PROMPT = 'You are a strict quality judge for task outputs.\n' +
+    'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
+    'On subsequent lines you may provide a one-sentence justification.\n' +
+    'Do not use any other format. Do not wrap the answer in code fences.\n' +
+    'If you reason step-by-step internally, still put PASS or FAIL on the first line of your final answer.';
+function parseDurationSafe(raw, fallback) {
+    if (raw == null)
+        return fallback;
+    const str = String(raw).trim();
+    const m = str.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
+    if (!m)
+        return fallback;
+    const n = Number(m[1]);
+    switch (m[2]) {
+        case 'ms': return n;
+        case 'm': return n * 60_000;
+        case 'h': return n * 3_600_000;
+        case 's':
+        default: return n * 1000;
+    }
+}
+// Head-and-tail truncation preserves the start of the output (where agents
+// usually declare intent) and the end (where they summarize results),
+// dropping the middle when the combined length exceeds the budget. This
+// keeps the judge's view of the output meaningful even for very long runs.
+function truncateForJudge(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    const marker = '\n...[truncated]...\n';
+    const budget = maxChars - marker.length;
+    if (budget <= 0)
+        return text.slice(0, maxChars);
+    const head = Math.floor(budget * 0.7);
+    const tail = budget - head;
+    return text.slice(0, head) + marker + text.slice(-tail);
+}
+// Strip reasoning-model thinking blocks so verdict parsing sees the real
+// answer. Qwen3 and DeepSeek-R1 both emit `<think>...</think>` inline in
+// message content when served via Ollama's OpenAI-compat route. We also
+// drop the legacy `<thinking>` variant some fine-tunes use. Applied
+// before any trimming so leading whitespace from the stripped block
+// doesn't leak into the first line.
+function stripThinking(content) {
+    return content
+        .replace(/<think>[\s\S]*?<\/think>/gi, '')
+        .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+        .trim();
+}
+async function callJudge(endpoint, model, apiKey, messages, timeoutMs, externalSignal) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const onExternalAbort = () => controller.abort();
+    if (externalSignal) {
+        if (externalSignal.aborted) {
+            controller.abort();
+        }
+        else {
+            externalSignal.addEventListener('abort', onExternalAbort, { once: true });
+        }
+    }
+    try {
+        const headers = {
+            'content-type': 'application/json',
+        };
+        // Only send Authorization when we actually have a key — local Ollama
+        // doesn't require one, and some OpenAI-compat proxies reject bogus
+        // placeholder tokens like "ollama" with 401 instead of ignoring them.
+        if (apiKey) {
+            headers.authorization = `Bearer ${apiKey}`;
+        }
+        const res = await fetch(endpoint, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                messages,
+                temperature: 0,
+                // `stream: false` is already the default but we set it explicitly
+                // because Ollama's OpenAI-compat route streams by default in some
+                // older versions.
+                stream: false,
+            }),
+            signal: controller.signal,
+        });
+        if (!res.ok) {
+            const text = await res.text().catch(() => '');
+            throw new Error(`judge endpoint ${res.status}: ${text.slice(0, 200)}`);
+        }
+        const payload = (await res.json());
+        const content = payload.choices?.[0]?.message?.content;
+        if (typeof content !== 'string') {
+            throw new Error('judge endpoint returned no message content');
+        }
+        return stripThinking(content);
+    }
+    finally {
+        clearTimeout(timer);
+        if (externalSignal)
+            externalSignal.removeEventListener('abort', onExternalAbort);
+    }
+}
+const LlmJudgeCompletion = {
+    name: 'llm_judge',
+    schema: {
+        description: 'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
+        fields: {
+            rubric: {
+                type: 'string',
+                required: true,
+                description: 'Criteria the judge should verify. Plain language.',
+                placeholder: 'Output must list at least 3 failing tests with file paths.',
+            },
+            model: {
+                type: 'string',
+                default: DEFAULT_MODEL,
+                description: 'Judge model name. Default is a small Ollama reasoning model (qwen3:4b). Swap for qwen3:8b, deepseek-r1:7b, or a hosted model for stricter judging.',
+                placeholder: DEFAULT_MODEL,
+            },
+            endpoint: {
+                type: 'string',
+                default: DEFAULT_ENDPOINT,
+                description: 'OpenAI-compatible chat completions endpoint. Defaults to local Ollama (http://localhost:11434/v1/chat/completions).',
+                placeholder: DEFAULT_ENDPOINT,
+            },
+            api_key_env: {
+                type: 'string',
+                description: 'Env var containing the bearer token for the judge endpoint. Leave unset for local Ollama; set to OPENAI_API_KEY etc. for hosted backends.',
+                placeholder: 'OPENAI_API_KEY',
+            },
+            timeout: {
+                type: 'duration',
+                default: '120s',
+                description: 'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
+            },
+            max_output_chars: {
+                type: 'number',
+                default: DEFAULT_MAX_OUTPUT_CHARS,
+                min: 500,
+                max: 200_000,
+                description: 'Truncate task stdout to this many chars before judging.',
+            },
+        },
+    },
+    async check(config, result, ctx) {
+        const rubric = config.rubric;
+        if (!rubric)
+            throw new Error('llm_judge completion: "rubric" is required');
+        // api_key_env is optional — when unset we talk to the endpoint
+        // anonymously (correct for local Ollama). When the user names an env
+        // var, we require it to be populated so config errors fail loudly
+        // instead of silently stripping auth.
+        const apiKeyEnv = config.api_key_env;
+        let apiKey;
+        if (apiKeyEnv) {
+            apiKey = process.env[apiKeyEnv];
+            if (!apiKey) {
+                throw new Error(`llm_judge completion: env var ${apiKeyEnv} is not set`);
+            }
+        }
+        const model = config.model ?? DEFAULT_MODEL;
+        const endpoint = config.endpoint ?? DEFAULT_ENDPOINT;
+        const timeoutMs = parseDurationSafe(config.timeout, DEFAULT_TIMEOUT_MS);
+        const maxChars = typeof config.max_output_chars === 'number' && config.max_output_chars > 0
+            ? Math.floor(config.max_output_chars)
+            : DEFAULT_MAX_OUTPUT_CHARS;
+        const userContent = `[Rubric]\n${rubric}\n\n` +
+            `[Exit Code]\n${result.exitCode}\n\n` +
+            `[Task Output]\n${truncateForJudge(result.stdout, maxChars)}`;
+        const messages = [
+            { role: 'system', content: SYSTEM_PROMPT },
+            { role: 'user', content: userContent },
+        ];
+        try {
+            const content = await callJudge(endpoint, model, apiKey, messages, timeoutMs, ctx.signal);
+            const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
+            const passed = firstLine.startsWith('PASS');
+            if (!passed) {
+                // Surface the judge's reasoning in logs so pipeline operators can
+                // see why an output was rejected without re-running it themselves.
+                console.warn(`[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`);
+            }
+            return passed;
+        }
+        catch (err) {
+            // Treat judge failures as FAIL: a completion gate that errors open
+            // is worse than one that errors closed. Operators can re-run the
+            // task once the judge endpoint is healthy again.
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
+            return false;
+        }
+    },
+};
+// ═══ Plugin self-description exports ═══
+export const pluginCategory = 'completions';
+export const pluginType = 'llm_judge';
+export default LlmJudgeCompletion;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAMtF,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,SAAS,iBAAiB,CAAC,GAAY,EAAE,QAAgB;IACvD,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACpD,IAAI,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACxB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;QACpB,KAAK,GAAG,CAAC,CAAE,OAAO,CAAC,GAAG,MAAM,CAAC;QAC7B,KAAK,GAAG,CAAC,CAAE,OAAO,CAAC,GAAG,SAAS,CAAC;QAChC,KAAK,GAAG,CAAC;QACT,OAAO,CAAC,CAAG,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,kBAAkB,GAAqB;IAC3C,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACzF,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE7B,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;QAEhE,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAC7B,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CACzD,CAAC;YACF,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,0CAA0C;AAC1C,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC;AAC5C,MAAM,CAAC,MAAM,UAAU,GAAG,WAAW,CAAC;AACtC,eAAe,kBAAkB,CAAC"}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@tagma/completion-llm-judge",
+  "version": "0.1.1",
+  "description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GoTagma/tagma-mono.git",
+    "directory": "packages/completion-llm-judge"
+  },
+  "bugs": "https://github.com/GoTagma/tagma-mono/issues",
+  "homepage": "https://github.com/GoTagma/tagma-mono/tree/main/packages/completion-llm-judge#readme",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "tagma-plugin",
+    "tagma",
+    "completion",
+    "llm-judge",
+    "llm-as-judge",
+    "openai"
+  ],
+  "engines": {
+    "bun": ">=1.3"
+  },
+  "tagmaPlugin": {
+    "category": "completions",
+    "type": "llm_judge"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "prepublishOnly": "bun run build"
+  },
+  "peerDependencies": {
+    "@tagma/types": "0.2.1"
+  },
+  "devDependencies": {
+    "bun-types": "^1.3.11",
+    "typescript": "^5.8.3"
+  }
+}

package/src/index.ts

@@ -0,0 +1,280 @@
+// ═══ LLM-as-Judge Completion Plugin ═══
+//
+// Uses an OpenAI-compatible chat completions endpoint to verify whether a
+// task's output satisfies a rubric. Complements the deterministic built-in
+// completions (`exit_code`, `file_exists`, `output_check`) with AI-powered
+// checks — useful when success is defined semantically rather than by a
+// grep-able pattern or a file on disk.
+//
+// Default backend is a **local Ollama** server using its OpenAI-compatible
+// route (`/v1/chat/completions`, available since Ollama 0.1.29+), with
+// `qwen3:4b` as a small, cheap-to-run reasoning model. No API key is
+// required for local Ollama; remote endpoints can set `api_key_env` to
+// whatever header the server expects.
+//
+// The judge is instructed to answer PASS/FAIL on the first line. Reasoning
+// models (qwen3, deepseek-r1, etc.) emit `<think>...</think>` blocks in
+// their message content — we strip those before parsing, so the rubric
+// works the same whether the judge model is a thinker or not.
+//
+// Usage in pipeline.yaml:
+//   plugins: ["@tagma/completion-llm-judge"]
+//   tracks:
+//     - tasks:
+//         - id: draft
+//           completion:
+//             type: llm_judge
+//             rubric: "Output must list at least 3 failing tests with file paths."
+//             # endpoint / model / api_key_env all default to local Ollama + qwen3:4b
+
+import type {
+  CompletionPlugin, CompletionContext, TaskResult,
+} from '@tagma/types';
+
+// Ollama exposes an OpenAI-compatible `/v1/chat/completions` route on port
+// 11434 by default. Point this at any OpenAI-compatible server (OpenAI,
+// vLLM, llama.cpp, LM Studio, Groq, Together, etc.) to swap backends.
+const DEFAULT_ENDPOINT = 'http://localhost:11434/v1/chat/completions';
+// qwen3:4b is a small reasoning model (~2.5 GB on disk, runs on CPU) that
+// reliably follows the PASS/FAIL-on-first-line instruction. Swap to
+// `qwen3:8b`, `deepseek-r1:7b`, or a hosted model for stricter judging.
+const DEFAULT_MODEL = 'qwen3:4b';
+const DEFAULT_TIMEOUT_MS = 120_000;
+const DEFAULT_MAX_OUTPUT_CHARS = 8_000;
+
+const SYSTEM_PROMPT =
+  'You are a strict quality judge for task outputs.\n' +
+  'Given the task rubric and actual output, answer on the FIRST LINE with exactly "PASS" or "FAIL".\n' +
+  'On subsequent lines you may provide a one-sentence justification.\n' +
+  'Do not use any other format. Do not wrap the answer in code fences.\n' +
+  'If you reason step-by-step internally, still put PASS or FAIL on the first line of your final answer.';
+
+interface ChatMessage {
+  readonly role: 'system' | 'user';
+  readonly content: string;
+}
+
+interface ChatCompletionResponse {
+  readonly choices?: ReadonlyArray<{
+    readonly message?: { readonly content?: string };
+  }>;
+}
+
+function parseDurationSafe(raw: unknown, fallback: number): number {
+  if (raw == null) return fallback;
+  const str = String(raw).trim();
+  const m = str.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/);
+  if (!m) return fallback;
+  const n = Number(m[1]);
+  switch (m[2]) {
+    case 'ms': return n;
+    case 'm':  return n * 60_000;
+    case 'h':  return n * 3_600_000;
+    case 's':
+    default:   return n * 1000;
+  }
+}
+
+// Head-and-tail truncation preserves the start of the output (where agents
+// usually declare intent) and the end (where they summarize results),
+// dropping the middle when the combined length exceeds the budget. This
+// keeps the judge's view of the output meaningful even for very long runs.
+function truncateForJudge(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  const marker = '\n...[truncated]...\n';
+  const budget = maxChars - marker.length;
+  if (budget <= 0) return text.slice(0, maxChars);
+  const head = Math.floor(budget * 0.7);
+  const tail = budget - head;
+  return text.slice(0, head) + marker + text.slice(-tail);
+}
+
+// Strip reasoning-model thinking blocks so verdict parsing sees the real
+// answer. Qwen3 and DeepSeek-R1 both emit `<think>...</think>` inline in
+// message content when served via Ollama's OpenAI-compat route. We also
+// drop the legacy `<thinking>` variant some fine-tunes use. Applied
+// before any trimming so leading whitespace from the stripped block
+// doesn't leak into the first line.
+function stripThinking(content: string): string {
+  return content
+    .replace(/<think>[\s\S]*?<\/think>/gi, '')
+    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+    .trim();
+}
+
+async function callJudge(
+  endpoint: string,
+  model: string,
+  apiKey: string | undefined,
+  messages: readonly ChatMessage[],
+  timeoutMs: number,
+  externalSignal: AbortSignal | undefined,
+): Promise<string> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  const onExternalAbort = (): void => controller.abort();
+  if (externalSignal) {
+    if (externalSignal.aborted) {
+      controller.abort();
+    } else {
+      externalSignal.addEventListener('abort', onExternalAbort, { once: true });
+    }
+  }
+
+  try {
+    const headers: Record<string, string> = {
+      'content-type': 'application/json',
+    };
+    // Only send Authorization when we actually have a key — local Ollama
+    // doesn't require one, and some OpenAI-compat proxies reject bogus
+    // placeholder tokens like "ollama" with 401 instead of ignoring them.
+    if (apiKey) {
+      headers.authorization = `Bearer ${apiKey}`;
+    }
+
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: 0,
+        // `stream: false` is already the default but we set it explicitly
+        // because Ollama's OpenAI-compat route streams by default in some
+        // older versions.
+        stream: false,
+      }),
+      signal: controller.signal,
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      throw new Error(`judge endpoint ${res.status}: ${text.slice(0, 200)}`);
+    }
+    const payload = (await res.json()) as ChatCompletionResponse;
+    const content = payload.choices?.[0]?.message?.content;
+    if (typeof content !== 'string') {
+      throw new Error('judge endpoint returned no message content');
+    }
+    return stripThinking(content);
+  } finally {
+    clearTimeout(timer);
+    if (externalSignal) externalSignal.removeEventListener('abort', onExternalAbort);
+  }
+}
+
+const LlmJudgeCompletion: CompletionPlugin = {
+  name: 'llm_judge',
+  schema: {
+    description:
+      'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
+    fields: {
+      rubric: {
+        type: 'string',
+        required: true,
+        description: 'Criteria the judge should verify. Plain language.',
+        placeholder: 'Output must list at least 3 failing tests with file paths.',
+      },
+      model: {
+        type: 'string',
+        default: DEFAULT_MODEL,
+        description:
+          'Judge model name. Default is a small Ollama reasoning model (qwen3:4b). Swap for qwen3:8b, deepseek-r1:7b, or a hosted model for stricter judging.',
+        placeholder: DEFAULT_MODEL,
+      },
+      endpoint: {
+        type: 'string',
+        default: DEFAULT_ENDPOINT,
+        description:
+          'OpenAI-compatible chat completions endpoint. Defaults to local Ollama (http://localhost:11434/v1/chat/completions).',
+        placeholder: DEFAULT_ENDPOINT,
+      },
+      api_key_env: {
+        type: 'string',
+        description:
+          'Env var containing the bearer token for the judge endpoint. Leave unset for local Ollama; set to OPENAI_API_KEY etc. for hosted backends.',
+        placeholder: 'OPENAI_API_KEY',
+      },
+      timeout: {
+        type: 'duration',
+        default: '120s',
+        description:
+          'Maximum time to wait for the judge response. Reasoning models need more time than chat models.',
+      },
+      max_output_chars: {
+        type: 'number',
+        default: DEFAULT_MAX_OUTPUT_CHARS,
+        min: 500,
+        max: 200_000,
+        description: 'Truncate task stdout to this many chars before judging.',
+      },
+    },
+  },
+
+  async check(
+    config: Record<string, unknown>,
+    result: TaskResult,
+    ctx: CompletionContext,
+  ): Promise<boolean> {
+    const rubric = config.rubric as string | undefined;
+    if (!rubric) throw new Error('llm_judge completion: "rubric" is required');
+
+    // api_key_env is optional — when unset we talk to the endpoint
+    // anonymously (correct for local Ollama). When the user names an env
+    // var, we require it to be populated so config errors fail loudly
+    // instead of silently stripping auth.
+    const apiKeyEnv = config.api_key_env as string | undefined;
+    let apiKey: string | undefined;
+    if (apiKeyEnv) {
+      apiKey = process.env[apiKeyEnv];
+      if (!apiKey) {
+        throw new Error(`llm_judge completion: env var ${apiKeyEnv} is not set`);
+      }
+    }
+
+    const model = (config.model as string | undefined) ?? DEFAULT_MODEL;
+    const endpoint = (config.endpoint as string | undefined) ?? DEFAULT_ENDPOINT;
+    const timeoutMs = parseDurationSafe(config.timeout, DEFAULT_TIMEOUT_MS);
+    const maxChars = typeof config.max_output_chars === 'number' && config.max_output_chars > 0
+      ? Math.floor(config.max_output_chars)
+      : DEFAULT_MAX_OUTPUT_CHARS;
+
+    const userContent =
+      `[Rubric]\n${rubric}\n\n` +
+      `[Exit Code]\n${result.exitCode}\n\n` +
+      `[Task Output]\n${truncateForJudge(result.stdout, maxChars)}`;
+
+    const messages: ChatMessage[] = [
+      { role: 'system', content: SYSTEM_PROMPT },
+      { role: 'user', content: userContent },
+    ];
+
+    try {
+      const content = await callJudge(
+        endpoint, model, apiKey, messages, timeoutMs, ctx.signal,
+      );
+      const firstLine = (content.split(/\r?\n/, 1)[0] ?? '').trim().toUpperCase();
+      const passed = firstLine.startsWith('PASS');
+      if (!passed) {
+        // Surface the judge's reasoning in logs so pipeline operators can
+        // see why an output was rejected without re-running it themselves.
+        console.warn(
+          `[llm_judge] verdict=${firstLine || '<empty>'} — full judge response:\n${content}`,
+        );
+      }
+      return passed;
+    } catch (err) {
+      // Treat judge failures as FAIL: a completion gate that errors open
+      // is worse than one that errors closed. Operators can re-run the
+      // task once the judge endpoint is healthy again.
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[llm_judge] judge call failed, marking task as not-complete: ${msg}`);
+      return false;
+    }
+  },
+};
+
+// ═══ Plugin self-description exports ═══
+export const pluginCategory = 'completions';
+export const pluginType = 'llm_judge';
+export default LlmJudgeCompletion;