@unotest/judge 0.24.0 → 0.26.0

package/dist/chunk-ADU4QPCJ.js

@@ -0,0 +1,1157 @@
+// src/errors.ts
+var JudgeError = class extends Error {
+  constructor(message, context = {}) {
+    super(message);
+    this.context = context;
+    this.name = new.target.name;
+  }
+  context;
+};
+var JudgeConfigError = class extends JudgeError {
+};
+var JudgeProviderError = class extends JudgeError {
+};
+var JudgeTransientError = class extends JudgeProviderError {
+};
+function judgeErrorCode(e) {
+  if (e instanceof JudgeConfigError) return "config";
+  if (e instanceof JudgeProviderError) return "provider-error";
+  return "internal";
+}
+function errorMessage(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+
+// src/providers/fake.ts
+var FAKE_MODEL_ID = "fake";
+var CONSTRAINT_RE = /^must(?<not>\s+not)?\s+contain:\s*(?<needle>.+)$/i;
+function parseFakeRubric(rubric) {
+  const constraints = [];
+  for (const line of rubric.split(/\r?\n/)) {
+    const m = CONSTRAINT_RE.exec(line.trim());
+    if (!m?.groups?.needle) continue;
+    constraints.push({
+      kind: m.groups.not ? "not-contains" : "contains",
+      needle: m.groups.needle.trim()
+    });
+  }
+  return constraints;
+}
+var FakeJudgeProvider = class {
+  async judgeOnce(request) {
+    const constraints = parseFakeRubric(request.rubric);
+    if (constraints.length === 0) {
+      return {
+        pass: false,
+        reasoning: 'fake provider: rubric has no parseable constraints \u2014 use lines like "must contain: <substring>" / "must not contain: <substring>", or switch to a real provider (UNOTEST_JUDGE_PROVIDER=vertex|claude|gemini|openai|anthropic)',
+        model: FAKE_MODEL_ID
+      };
+    }
+    const text = request.text.toLowerCase();
+    const violations = [];
+    for (const c of constraints) {
+      const hit = text.includes(c.needle.toLowerCase());
+      if (c.kind === "contains" && !hit) violations.push(`missing required "${c.needle}"`);
+      if (c.kind === "not-contains" && hit) violations.push(`contains forbidden "${c.needle}"`);
+    }
+    if (violations.length > 0) {
+      return { pass: false, reasoning: violations.join("; "), model: FAKE_MODEL_ID };
+    }
+    return {
+      pass: true,
+      reasoning: `all ${constraints.length} constraint(s) satisfied`,
+      model: FAKE_MODEL_ID
+    };
+  }
+};
+
+// src/verdict.ts
+var VERDICT_PROMPT = (rubric, text) => `You are a strict test judge. Evaluate the TEXT against the RUBRIC.
+Reply with ONLY a raw JSON object, no markdown code fences: {"verdict": "pass" | "fail", "reasoning": "<one short sentence>"}.
+RUBRIC:
+${rubric}
+
+TEXT:
+${text}`;
+var FENCED_JSON = /^```(?:json)?\s*\n([\s\S]*?)\n\s*```$/;
+function parseVerdictReply(text, model) {
+  const trimmed = text.trim();
+  const parsed = parseJsonish(FENCED_JSON.exec(trimmed)?.[1] ?? trimmed);
+  if (parsed === void 0) {
+    throw new JudgeProviderError(
+      `model "${model}" did not reply with the requested JSON verdict: ${text.slice(0, 200)}`,
+      { model, raw: text }
+    );
+  }
+  const { verdict, reasoning } = parsed;
+  const normalized = typeof verdict === "string" ? verdict.trim().toLowerCase() : verdict;
+  if (normalized !== "pass" && normalized !== "fail") {
+    throw new JudgeProviderError(
+      `model "${model}" replied with an unknown verdict ${JSON.stringify(verdict)} (expected "pass"/"fail")`,
+      { model, raw: text }
+    );
+  }
+  return {
+    pass: normalized === "pass",
+    reasoning: typeof reasoning === "string" && reasoning.length > 0 ? reasoning : "(no reasoning)",
+    raw: text
+  };
+}
+function parseJsonish(candidate) {
+  const spans = [candidate];
+  const start = candidate.indexOf("{");
+  const end = candidate.lastIndexOf("}");
+  if (start >= 0 && end > start) spans.push(candidate.slice(start, end + 1));
+  for (const span of spans) {
+    try {
+      const parsed = JSON.parse(span);
+      if (typeof parsed === "object" && parsed !== null) return parsed;
+    } catch {
+    }
+  }
+  return void 0;
+}
+
+// src/providers/vertex.ts
+import { JUDGE_ENV } from "@unotest/protocol";
+
+// src/providers/adc-auth.ts
+async function buildAdcTokenSource() {
+  const specifier = "google-auth-library";
+  let mod;
+  try {
+    mod = await import(specifier);
+  } catch (e) {
+    throw new JudgeConfigError(
+      `Vertex provider could not load "google-auth-library" (a dependency of this package \u2014 a broken install?): ${e instanceof Error ? e.message : String(e)}. Workaround: pass a token via UNOTEST_JUDGE_ACCESS_TOKEN (\`gcloud auth print-access-token\`)`
+    );
+  }
+  const auth = new mod.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/cloud-platform"] });
+  return async () => {
+    let token;
+    try {
+      token = await auth.getAccessToken();
+    } catch (e) {
+      throw describeAdcFailure(e);
+    }
+    if (!token) {
+      throw new JudgeConfigError(
+        "ADC produced no access token \u2014 run `gcloud auth application-default login` or set GOOGLE_APPLICATION_CREDENTIALS"
+      );
+    }
+    return token;
+  };
+}
+var REAUTH_CODES = /* @__PURE__ */ new Set(["invalid_grant", "invalid_rapt"]);
+function describeAdcFailure(e) {
+  const body = oauthErrorBody(e);
+  if (body && (REAUTH_CODES.has(body.error) || REAUTH_CODES.has(body.error_subtype ?? ""))) {
+    return new JudgeConfigError(
+      `Application Default Credentials need re-authentication (${body.error}${body.error_subtype ? `/${body.error_subtype}` : ""}) \u2014 run \`gcloud auth application-default login\``,
+      { oauthError: body.error }
+    );
+  }
+  const detail = body?.error_description ?? body?.error ?? (e instanceof Error ? e.message : String(e));
+  return new JudgeConfigError(
+    `Application Default Credentials failed: ${detail} \u2014 check GOOGLE_CLOUD_PROJECT / GOOGLE_CLOUD_LOCATION and \`gcloud auth application-default login\``
+  );
+}
+function oauthErrorBody(e) {
+  if (typeof e !== "object" || e === null) return void 0;
+  const data = e.response?.data;
+  if (typeof data !== "object" || data === null) return void 0;
+  const body = data;
+  return typeof body.error === "string" ? body : void 0;
+}
+
+// src/providers/generate-content.ts
+function generateContentBody(request) {
+  return {
+    contents: [
+      { role: "user", parts: [{ text: VERDICT_PROMPT(request.rubric, request.text) }] }
+    ],
+    generationConfig: {
+      temperature: 0,
+      responseMimeType: "application/json"
+    }
+  };
+}
+function extractCandidateText(body, model, backend) {
+  let parsed;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    throw new JudgeProviderError(`${backend} reply is not JSON: ${body.slice(0, 200)}`, { model });
+  }
+  const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
+  if (typeof text !== "string" || text.length === 0) {
+    throw new JudgeProviderError(
+      `${backend} reply carries no candidate text (blocked or empty): ${body.slice(0, 300)}`,
+      { model }
+    );
+  }
+  return text;
+}
+
+// src/providers/http-json.ts
+var MAX_RETRY_AFTER_MS = 1e4;
+function postJson(call) {
+  return send(call, {
+    method: "POST",
+    headers: { "content-type": "application/json", ...call.headers },
+    body: JSON.stringify(call.body),
+    signal: AbortSignal.timeout(call.timeoutMs)
+  });
+}
+async function getOk(call) {
+  await send(call, {
+    method: "GET",
+    headers: call.headers,
+    signal: AbortSignal.timeout(call.timeoutMs)
+  });
+}
+var HttpBackend = class {
+  constructor(spec) {
+    this.spec = spec;
+  }
+  spec;
+  post(url, body, model, headers = {}) {
+    return postJson({ ...this.spec, headers: { ...this.spec.headers, ...headers }, url, body, model });
+  }
+  /** Ask a free endpoint whether the credentials are still good. */
+  async probe(url, headers = {}) {
+    await getOk({ ...this.spec, headers: { ...this.spec.headers, ...headers }, url });
+  }
+};
+function apiBackend(backend, authHint2, headers, opts) {
+  return new HttpBackend({
+    backend,
+    authHint: authHint2,
+    headers,
+    timeoutMs: opts.timeoutMs,
+    fetchImpl: opts.fetchImpl ?? fetch
+  });
+}
+async function send(call, init) {
+  const { url, backend, fetchImpl } = call;
+  let res;
+  try {
+    res = await fetchImpl(url, init);
+  } catch (e) {
+    throw new JudgeTransientError(
+      `cannot reach ${backend} at ${url} (${e instanceof Error ? e.message : String(e)})`,
+      { url }
+    );
+  }
+  const text = await res.text();
+  if (res.ok) return text;
+  throw describeFailure(call, res, text);
+}
+function describeFailure(call, res, body) {
+  const { url, backend, authHint: authHint2 } = call;
+  if (res.status === 401 || res.status === 403) {
+    return new JudgeConfigError(
+      `${backend} rejected the credentials (HTTP ${res.status}) \u2014 ${authHint2}`,
+      { status: res.status, url }
+    );
+  }
+  if (res.status === 429 || res.status >= 500) {
+    const retryAfterMs = retryAfter(res);
+    return new JudgeTransientError(
+      `${backend} replied ${res.status} (${res.status === 429 ? "rate limited" : "backend error"})`,
+      { status: res.status, url, ...retryAfterMs === void 0 ? {} : { retryAfterMs } }
+    );
+  }
+  if (!looksLikeJson(body)) {
+    return new JudgeProviderError(
+      `${backend} replied ${res.status} with a non-JSON body \u2014 the endpoint is wrong, not the model: ${url} (UNOTEST_JUDGE_LOG_LEVEL=debug logs the body)`,
+      { status: res.status, url, body: body.slice(0, 2e3) }
+    );
+  }
+  return new JudgeProviderError(
+    `${backend} replied ${res.status}: ${body.slice(0, 500)}`,
+    { status: res.status, url }
+  );
+}
+function looksLikeJson(body) {
+  const head = body.trimStart()[0];
+  return head === "{" || head === "[";
+}
+function retryAfter(res) {
+  const raw = res.headers.get("retry-after");
+  if (!raw) return void 0;
+  const seconds = Number.parseFloat(raw);
+  const ms = Number.isFinite(seconds) ? seconds * 1e3 : Date.parse(raw) - Date.now();
+  if (!Number.isFinite(ms) || ms <= 0) return void 0;
+  return Math.min(ms, MAX_RETRY_AFTER_MS);
+}
+
+// src/providers/vertex.ts
+function vertexHost(location) {
+  return location === "global" ? "https://aiplatform.googleapis.com" : `https://${location}-aiplatform.googleapis.com`;
+}
+function modelPath(model) {
+  return model.includes("/") ? model : `publishers/google/models/${model}`;
+}
+function vertexEndpoint(project, location, model) {
+  return `${vertexHost(location)}/v1/projects/${project}/locations/${location}/${modelPath(model)}:generateContent`;
+}
+var TOKENINFO_URL = "https://oauth2.googleapis.com/tokeninfo";
+var TOKEN_ENV = JUDGE_ENV.accessToken;
+function authHint(opts) {
+  return opts.accessToken ? "UNOTEST_JUDGE_ACCESS_TOKEN has expired (these tokens live about an hour) \u2014 refresh it with `gcloud auth print-access-token`, or unset it to use ADC, which refreshes itself" : "run `gcloud auth application-default login`, and check that GOOGLE_CLOUD_PROJECT has Vertex AI enabled";
+}
+var VertexJudgeProvider = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.api = apiBackend("Vertex AI", authHint(opts), {}, opts);
+  }
+  opts;
+  api;
+  tokenSource;
+  async judgeOnce(request) {
+    const { project, location, model } = this.opts;
+    const token = await this.accessToken();
+    const body = await this.api.post(
+      vertexEndpoint(project, location, model),
+      generateContentBody(request),
+      model,
+      { authorization: `Bearer ${token}` }
+    );
+    return { ...parseVerdictReply(extractCandidateText(body, model, "Vertex AI"), model), model };
+  }
+  /** Two credentials, two free probes. ADC: fetching a token exercises the
+   *  exact thing that dies on an org's daily reauth policy. A static token:
+   *  Google's own `tokeninfo` says whether it is still alive — and a static
+   *  token lives about an hour, so "valid when the service started" is not
+   *  the same question as "valid now". */
+  async checkAuth() {
+    const { accessToken } = this.opts;
+    if (!accessToken) {
+      await this.accessToken();
+      return;
+    }
+    try {
+      await this.api.probe(`${TOKENINFO_URL}?access_token=${encodeURIComponent(accessToken)}`);
+    } catch (e) {
+      if (e instanceof JudgeTransientError) return;
+      throw new JudgeConfigError(`Google rejected ${TOKEN_ENV} \u2014 ${authHint(this.opts)}`);
+    }
+  }
+  async accessToken() {
+    if (this.opts.accessToken) return this.opts.accessToken;
+    this.tokenSource ??= await buildAdcTokenSource();
+    return this.tokenSource();
+  }
+};
+
+// src/providers/gemini.ts
+var BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+var AUTH_HINT = "check GEMINI_API_KEY (https://aistudio.google.com/apikey)";
+function modelId(model) {
+  return model.replace(/^models\//, "");
+}
+var GeminiJudgeProvider = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.api = apiBackend("Gemini API", AUTH_HINT, { "x-goog-api-key": opts.apiKey }, opts);
+  }
+  opts;
+  api;
+  async judgeOnce(request) {
+    const { model } = this.opts;
+    const body = await this.api.post(
+      `${BASE_URL}/models/${modelId(model)}:generateContent`,
+      generateContentBody(request),
+      model
+    );
+    return { ...parseVerdictReply(extractCandidateText(body, model, "Gemini API"), model), model };
+  }
+  checkAuth() {
+    return this.api.probe(`${BASE_URL}/models?pageSize=1`);
+  }
+};
+
+// src/providers/openai.ts
+var BASE_URL2 = "https://api.openai.com/v1";
+var AUTH_HINT2 = "check OPENAI_API_KEY (https://platform.openai.com/api-keys)";
+var OpenAiJudgeProvider = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.api = apiBackend("OpenAI", AUTH_HINT2, { authorization: `Bearer ${opts.apiKey}` }, opts);
+  }
+  opts;
+  api;
+  async judgeOnce(request) {
+    const { model } = this.opts;
+    const body = await this.api.post(
+      `${BASE_URL2}/chat/completions`,
+      {
+        model,
+        messages: [{ role: "user", content: VERDICT_PROMPT(request.rubric, request.text) }],
+        response_format: { type: "json_object" }
+      },
+      model
+    );
+    return { ...parseVerdictReply(extractMessageContent(body, model), model), model };
+  }
+  /** `GET /models` is free and needs the same key the verdict call uses. */
+  checkAuth() {
+    return this.api.probe(`${BASE_URL2}/models`);
+  }
+};
+function extractMessageContent(body, model) {
+  let parsed;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    throw new JudgeProviderError(`OpenAI reply is not JSON: ${body.slice(0, 200)}`, { model });
+  }
+  const choice = parsed.choices?.[0];
+  if (choice?.finish_reason === "length") {
+    throw new JudgeProviderError(
+      `OpenAI model "${model}" hit its token limit before finishing the verdict \u2014 the verdict itself is short, so this usually means a reasoning model spent the budget thinking: pick a non-reasoning model via UNOTEST_JUDGE_MODEL`,
+      { model, finishReason: "length" }
+    );
+  }
+  const content = choice?.message?.content;
+  if (typeof content !== "string" || content.length === 0) {
+    throw new JudgeProviderError(
+      `OpenAI reply carries no message content: ${body.slice(0, 300)}`,
+      { model }
+    );
+  }
+  return content;
+}
+
+// src/providers/anthropic.ts
+var BASE_URL3 = "https://api.anthropic.com/v1";
+var ANTHROPIC_VERSION = "2023-06-01";
+var MAX_TOKENS = 1024;
+var AUTH_HINT3 = "check ANTHROPIC_API_KEY (https://console.anthropic.com/settings/keys)";
+var AnthropicJudgeProvider = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.api = apiBackend(
+      "Anthropic API",
+      AUTH_HINT3,
+      { "x-api-key": opts.apiKey, "anthropic-version": ANTHROPIC_VERSION },
+      opts
+    );
+  }
+  opts;
+  api;
+  async judgeOnce(request) {
+    const { model } = this.opts;
+    const body = await this.api.post(
+      `${BASE_URL3}/messages`,
+      {
+        model,
+        max_tokens: MAX_TOKENS,
+        messages: [{ role: "user", content: VERDICT_PROMPT(request.rubric, request.text) }]
+      },
+      model
+    );
+    return { ...parseVerdictReply(extractTextBlock(body, model), model), model };
+  }
+  /** `GET /models` is free and needs the same key the verdict call uses. */
+  checkAuth() {
+    return this.api.probe(`${BASE_URL3}/models?limit=1`);
+  }
+};
+function extractTextBlock(body, model) {
+  let parsed;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    throw new JudgeProviderError(`Anthropic API reply is not JSON: ${body.slice(0, 200)}`, {
+      model
+    });
+  }
+  const reply = parsed;
+  if (reply.stop_reason === "max_tokens") {
+    throw new JudgeProviderError(
+      `Anthropic model "${model}" hit the ${MAX_TOKENS}-token limit before finishing the verdict \u2014 a reasoning model spends the budget thinking: pick a non-reasoning model via UNOTEST_JUDGE_MODEL`,
+      { model, stopReason: "max_tokens" }
+    );
+  }
+  const text = reply.content?.find((b) => b.type === "text")?.text;
+  if (typeof text === "string" && text.length > 0) return text;
+  throw new JudgeProviderError(
+    `Anthropic API reply carries no text block (refusal or empty): ${body.slice(0, 300)}`,
+    { model }
+  );
+}
+
+// src/providers/run-process.ts
+import { spawn } from "child_process";
+var runProcess = (file, args, opts) => new Promise((resolve, reject) => {
+  const child = spawn(file, args);
+  let stdout = "";
+  let stderr = "";
+  let overflowed = false;
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    child.kill();
+  }, opts.timeoutMs);
+  const settle = (finish) => {
+    clearTimeout(timer);
+    finish();
+  };
+  child.stdout.on("data", (chunk) => {
+    if (stdout.length + chunk.length > opts.maxBuffer) {
+      overflowed = true;
+      child.kill();
+      return;
+    }
+    stdout += chunk.toString("utf8");
+  });
+  child.stderr.on("data", (chunk) => {
+    stderr += chunk.toString("utf8");
+  });
+  child.on("error", (e) => settle(() => reject(Object.assign(e, { stderr }))));
+  child.on(
+    "close",
+    (code, signal) => settle(() => {
+      if (overflowed) {
+        reject(
+          Object.assign(
+            new Error(`${file} produced more than ${opts.maxBuffer} bytes of output`),
+            { code: "ERR_MAXBUFFER", stderr }
+          )
+        );
+        return;
+      }
+      if (timedOut || signal !== null) {
+        reject(
+          Object.assign(new Error(`${file} was killed (${signal ?? "timeout"})`), {
+            killed: true,
+            stderr
+          })
+        );
+        return;
+      }
+      if (code !== 0) {
+        reject(Object.assign(new Error(`${file} exited with code ${code}`), { code, stderr }));
+        return;
+      }
+      resolve({ stdout, stderr });
+    })
+  );
+  child.stdin.on("error", () => {
+  });
+  if (opts.input !== void 0) child.stdin.end(opts.input);
+  else child.stdin.end();
+});
+
+// src/providers/claude-cli.ts
+var MAX_STDOUT_BYTES = 10 * 1024 * 1024;
+var VERSION_TIMEOUT_MS = 15e3;
+var ClaudeCliJudgeProvider = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.run = opts.runImpl ?? runProcess;
+  }
+  opts;
+  run;
+  async judgeOnce(request) {
+    const { bin, model, timeoutMs } = this.opts;
+    const args = [
+      "-p",
+      "--output-format",
+      "json",
+      "--strict-mcp-config",
+      ...model ? ["--model", model] : []
+    ];
+    let stdout;
+    try {
+      ({ stdout } = await this.run(bin, args, {
+        timeoutMs,
+        maxBuffer: MAX_STDOUT_BYTES,
+        input: VERDICT_PROMPT(request.rubric, request.text)
+      }));
+    } catch (e) {
+      throw toTypedError(e, bin, timeoutMs);
+    }
+    const envelope = parseEnvelope(stdout, bin);
+    const modelLabel = firstModelId(envelope) ?? model ?? "claude";
+    if (envelope.is_error === true || typeof envelope.result !== "string" || !envelope.result) {
+      throw new JudgeProviderError(
+        `${bin} -p returned an error result: ${String(envelope.result ?? stdout).slice(0, 300)}`,
+        { model: modelLabel }
+      );
+    }
+    return { ...parseVerdictReply(envelope.result, modelLabel), model: modelLabel };
+  }
+  /** The binary exists and runs. It cannot prove the session is still
+   *  authenticated without spending a model call, so that failure stays a
+   *  first-verdict one — but "claude is not installed on this box" is the
+   *  common case and it is caught here. */
+  async checkAuth() {
+    const { bin } = this.opts;
+    try {
+      await this.run(bin, ["--version"], {
+        timeoutMs: VERSION_TIMEOUT_MS,
+        maxBuffer: MAX_STDOUT_BYTES
+      });
+    } catch (e) {
+      throw toTypedError(e, bin, VERSION_TIMEOUT_MS);
+    }
+  }
+};
+var TRANSIENT_SPAWN_CODES = /* @__PURE__ */ new Set(["EAGAIN", "ENOMEM", "EMFILE", "ENFILE"]);
+function toTypedError(e, bin, timeoutMs) {
+  const err = e;
+  if (typeof err.code === "string" && TRANSIENT_SPAWN_CODES.has(err.code)) {
+    return new JudgeTransientError(`${bin} could not be spawned (${err.code})`, { bin });
+  }
+  if (err.code === "ENOENT") {
+    return new JudgeConfigError(
+      `Claude Code CLI not found ("${bin}") \u2014 install it (https://claude.com/claude-code) or point UNOTEST_JUDGE_CLAUDE_BIN at the binary`,
+      { bin }
+    );
+  }
+  if (err.code === "EACCES") {
+    return new JudgeConfigError(`Claude Code CLI is not executable ("${bin}")`, { bin });
+  }
+  if (err.killed === true) {
+    return new JudgeProviderError(`${bin} -p timed out after ${timeoutMs}ms`, { bin, timeoutMs });
+  }
+  const stderr = typeof err.stderr === "string" && err.stderr ? err.stderr : String(err.message ?? e);
+  return new JudgeProviderError(`${bin} -p failed: ${stderr.slice(0, 500)}`, { bin });
+}
+function parseEnvelope(stdout, bin) {
+  try {
+    return JSON.parse(stdout);
+  } catch {
+    throw new JudgeProviderError(
+      `${bin} -p did not return the JSON envelope: ${stdout.slice(0, 300)}`,
+      { bin }
+    );
+  }
+}
+function firstModelId(envelope) {
+  const keys = envelope.modelUsage ? Object.keys(envelope.modelUsage) : [];
+  return keys.length > 0 ? keys[0] : void 0;
+}
+
+// src/policy.ts
+async function judgeWithRetries(provider, request, retries) {
+  const maxAttempts = Math.max(0, retries) + 1;
+  let last = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const single = await provider.judgeOnce(request);
+    last = {
+      verdict: single.pass ? "pass" : "fail",
+      reasoning: single.reasoning,
+      model: single.model,
+      attempts: attempt
+    };
+    if (single.pass) return last;
+  }
+  return last;
+}
+async function judgeWithVote(provider, request, votes) {
+  const settled = await Promise.allSettled(
+    Array.from({ length: votes }, () => provider.judgeOnce(request))
+  );
+  const ballots = settled.flatMap((s) => s.status === "fulfilled" ? [s.value] : []);
+  const failure = settled.find((s) => s.status === "rejected")?.reason;
+  const quorum = Math.floor(votes / 2) + 1;
+  if (ballots.length < quorum) throw failure;
+  const passes = ballots.filter((b) => b.pass);
+  const fails = ballots.filter((b) => !b.pass);
+  const winners = passes.length > fails.length ? passes : fails;
+  if (winners.length * 2 <= ballots.length) throw failure;
+  const winner = winners[0];
+  const lost = votes - ballots.length;
+  return {
+    verdict: winner.pass ? "pass" : "fail",
+    reasoning: `majority ${winners.length}/${votes}` + (lost > 0 ? ` (${lost} ballot(s) errored)` : "") + `: ${winner.reasoning}`,
+    model: winner.model,
+    attempts: votes
+  };
+}
+
+// src/logger.ts
+var LOG_LEVELS = ["silent", "error", "warn", "info", "debug"];
+var ConsoleLogger = class {
+  constructor(level, out = (l) => process.stdout.write(`${l}
+`), err = (l) => process.stderr.write(`${l}
+`)) {
+    this.level = level;
+    this.out = out;
+    this.err = err;
+  }
+  level;
+  out;
+  err;
+  error(message) {
+    if (this.enabled("error")) this.err(message);
+  }
+  warn(message) {
+    if (this.enabled("warn")) this.err(message);
+  }
+  info(message) {
+    if (this.enabled("info")) this.out(message);
+  }
+  debug(message) {
+    if (this.enabled("debug")) this.out(message);
+  }
+  enabled(of) {
+    return LOG_LEVELS.indexOf(this.level) >= LOG_LEVELS.indexOf(of);
+  }
+};
+var SILENT_LOGGER = {
+  error: () => {
+  },
+  warn: () => {
+  },
+  info: () => {
+  },
+  debug: () => {
+  }
+};
+
+// src/logging-provider.ts
+var LoggingJudgeProvider = class {
+  constructor(inner, logger, nowFn = Date.now) {
+    this.inner = inner;
+    this.logger = logger;
+    this.nowFn = nowFn;
+    if (inner.checkAuth) this.checkAuth = () => inner.checkAuth();
+  }
+  inner;
+  logger;
+  nowFn;
+  /** Mirrors the inner provider's capability instead of faking a probe the
+   *  wrapped one does not have. */
+  checkAuth;
+  async judgeOnce(request) {
+    this.logger.debug(`judge: effective prompt
+${VERDICT_PROMPT(request.rubric, request.text)}`);
+    const started = this.nowFn();
+    try {
+      const verdict = await this.inner.judgeOnce(request);
+      this.logger.debug(
+        `judge: ballot ${verdict.pass ? "pass" : "fail"} ${verdict.model} ${this.nowFn() - started}ms \u2014 ${verdict.reasoning}`
+      );
+      if (verdict.raw !== void 0) this.logger.debug(`judge: raw reply
+${verdict.raw}`);
+      return verdict;
+    } catch (e) {
+      this.logger.debug(
+        `judge: ballot ERROR after ${this.nowFn() - started}ms \u2014 ${e instanceof Error ? e.message : String(e)}` + (e instanceof JudgeError ? `
+context: ${JSON.stringify(e.context)}` : "")
+      );
+      throw e;
+    }
+  }
+};
+function withLogging(provider, logger) {
+  return logger ? new LoggingJudgeProvider(provider, logger) : provider;
+}
+
+// src/transient-retry.ts
+var ATTEMPTS = 3;
+var BACKOFF_MS = [500, 1500];
+var realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var TransientRetryProvider = class {
+  constructor(inner, opts = {}) {
+    this.inner = inner;
+    this.opts = opts;
+    if (inner.checkAuth) this.checkAuth = () => this.retry(() => inner.checkAuth());
+  }
+  inner;
+  opts;
+  /** Mirrors the inner provider's capability rather than inventing one. */
+  checkAuth;
+  judgeOnce(request) {
+    return this.retry(() => this.inner.judgeOnce(request));
+  }
+  async retry(attempt) {
+    const sleep = this.opts.sleepImpl ?? realSleep;
+    for (let i = 1; ; i++) {
+      try {
+        return await attempt();
+      } catch (e) {
+        if (!(e instanceof JudgeTransientError) || i >= ATTEMPTS) throw e;
+        const after = e.context.retryAfterMs;
+        await sleep(typeof after === "number" ? after : BACKOFF_MS[i - 1]);
+      }
+    }
+  }
+};
+function withTransientRetry(provider, opts) {
+  return new TransientRetryProvider(provider, opts);
+}
+
+// src/auth-probe.ts
+var DEFAULT_TTL_MS = 15e3;
+function createAuthProbe(service) {
+  return new AuthProbe(service.checkAuth ? { checkAuth: () => service.checkAuth() } : {});
+}
+var AuthProbe = class {
+  constructor(opts) {
+    this.opts = opts;
+  }
+  opts;
+  cached;
+  /** @param fresh bypass the cache (startup preflight, `--check`). */
+  async status(fresh = false) {
+    const now = (this.opts.nowFn ?? Date.now)();
+    const ttl = this.opts.ttlMs ?? DEFAULT_TTL_MS;
+    if (!fresh && this.cached && now - this.cached.at < ttl) return this.cached.result;
+    const result = await this.run();
+    this.cached = { at: now, result };
+    return result;
+  }
+  async run() {
+    if (!this.opts.checkAuth) return { ok: true };
+    try {
+      await this.opts.checkAuth();
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, code: judgeErrorCode(e), error: errorMessage(e) };
+    }
+  }
+};
+
+// src/env.ts
+import { JUDGE_ENV as JUDGE_ENV2 } from "@unotest/protocol";
+var PROVIDER_KINDS = [
+  "fake",
+  "vertex",
+  "claude",
+  "gemini",
+  "openai",
+  "anthropic"
+];
+var DEFAULT_MODEL = {
+  fake: void 0,
+  vertex: "gemini-2.5-flash",
+  gemini: "gemini-2.5-flash",
+  openai: "gpt-5-mini",
+  anthropic: "claude-haiku-4-5",
+  claude: void 0
+};
+var API_KEY_ENV = {
+  gemini: "GEMINI_API_KEY",
+  openai: "OPENAI_API_KEY",
+  anthropic: "ANTHROPIC_API_KEY"
+};
+var DEFAULT_RETRIES = 1;
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_CLAUDE_TIMEOUT_MS = 12e4;
+function intEnv(env, name, fallback, min, max) {
+  const raw = env[name];
+  if (raw === void 0 || raw === "") return fallback;
+  const n = Number.parseInt(raw, 10);
+  if (Number.isNaN(n) || n < min || n > max) {
+    throw new JudgeConfigError(`${name} must be an integer in [${min}, ${max}], got "${raw}"`);
+  }
+  return n;
+}
+function resolveJudgeServiceEnv(env) {
+  const provider = env[JUDGE_ENV2.provider];
+  if (!provider || !PROVIDER_KINDS.includes(provider)) {
+    const list = PROVIDER_KINDS.map((k) => `"${k}"`).join(" | ");
+    throw new JudgeConfigError(
+      provider === void 0 || provider === "" ? `${JUDGE_ENV2.provider} is not set \u2014 one of ${list} ("fake" is deterministic and CI-safe)` : `${JUDGE_ENV2.provider} must be one of ${list}, got "${provider}"`
+    );
+  }
+  const votes = intEnv(env, JUDGE_ENV2.vote, 1, 1, 9);
+  if (votes % 2 === 0) {
+    throw new JudgeConfigError(
+      `${JUDGE_ENV2.vote} must be odd (1, 3, 5, 7, 9) \u2014 an even vote can tie, and a tie has no verdict; got "${votes}"`
+    );
+  }
+  const resolved = {
+    provider,
+    votes,
+    retries: intEnv(env, JUDGE_ENV2.retries, DEFAULT_RETRIES, 0, 10),
+    timeoutMs: intEnv(
+      env,
+      JUDGE_ENV2.callTimeoutMs,
+      provider === "claude" ? DEFAULT_CLAUDE_TIMEOUT_MS : DEFAULT_TIMEOUT_MS,
+      1e3,
+      6e5
+    )
+  };
+  const model = env[JUDGE_ENV2.model] || DEFAULT_MODEL[provider];
+  if (model) resolved.model = model;
+  if (provider === "vertex") {
+    const project = env.GOOGLE_CLOUD_PROJECT;
+    const location = env.GOOGLE_CLOUD_LOCATION;
+    if (!project || !location) {
+      throw new JudgeConfigError(
+        `${JUDGE_ENV2.provider}=vertex requires GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION (ADC auth, no API keys). Location is "global" or a region such as "us-central1" \u2014 pick it together with the model: model availability differs per location`
+      );
+    }
+    resolved.project = project;
+    resolved.location = location;
+    const accessToken = env[JUDGE_ENV2.accessToken];
+    if (accessToken) resolved.accessToken = accessToken;
+  }
+  const keyEnvName = API_KEY_ENV[provider];
+  if (keyEnvName) {
+    const apiKey = env[keyEnvName];
+    if (!apiKey) {
+      throw new JudgeConfigError(`${JUDGE_ENV2.provider}=${provider} requires ${keyEnvName}`);
+    }
+    resolved.apiKey = apiKey;
+  }
+  if (provider === "claude") {
+    resolved.claudeBin = env[JUDGE_ENV2.claudeBin] || "claude";
+  }
+  return resolved;
+}
+function resolveJudgeServerEnv(env) {
+  const resolved = {
+    host: env[JUDGE_ENV2.host] || "127.0.0.1",
+    port: intEnv(env, JUDGE_ENV2.port, 8790, 1, 65535),
+    skipPreflight: boolEnv(env, JUDGE_ENV2.skipPreflight)
+  };
+  const token = env[JUDGE_ENV2.token];
+  if (token) resolved.token = token;
+  return resolved;
+}
+function resolveLogLevel(env) {
+  const raw = (env[JUDGE_ENV2.logLevel] || env.UNOTEST_LOG_LEVEL || "").trim().toLowerCase();
+  if (raw === "") return void 0;
+  if (!LOG_LEVELS.includes(raw)) {
+    throw new JudgeConfigError(
+      `${JUDGE_ENV2.logLevel} must be one of ${LOG_LEVELS.join(" | ")}, got "${raw}"`
+    );
+  }
+  return raw;
+}
+function boolEnv(env, name) {
+  const raw = (env[name] || "").trim().toLowerCase();
+  return raw === "1" || raw === "true" || raw === "yes";
+}
+
+// src/judge-service.ts
+function buildProvider(env) {
+  switch (env.provider) {
+    case "fake":
+      return new FakeJudgeProvider();
+    case "vertex":
+      return new VertexJudgeProvider({
+        project: env.project,
+        location: env.location,
+        model: env.model,
+        timeoutMs: env.timeoutMs,
+        ...env.accessToken ? { accessToken: env.accessToken } : {}
+      });
+    case "gemini":
+      return new GeminiJudgeProvider({
+        apiKey: env.apiKey,
+        model: env.model,
+        timeoutMs: env.timeoutMs
+      });
+    case "openai":
+      return new OpenAiJudgeProvider({
+        apiKey: env.apiKey,
+        model: env.model,
+        timeoutMs: env.timeoutMs
+      });
+    case "anthropic":
+      return new AnthropicJudgeProvider({
+        apiKey: env.apiKey,
+        model: env.model,
+        timeoutMs: env.timeoutMs
+      });
+    case "claude":
+      return new ClaudeCliJudgeProvider({
+        bin: env.claudeBin,
+        timeoutMs: env.timeoutMs,
+        ...env.model ? { model: env.model } : {}
+      });
+  }
+}
+function createJudgeService(env, provider, logger) {
+  const p = withTransientRetry(withLogging(provider ?? buildProvider(env), logger));
+  const service = {
+    judge: (request) => env.votes > 1 ? judgeWithVote(p, request, env.votes) : judgeWithRetries(p, request, env.retries)
+  };
+  if (p.checkAuth) service.checkAuth = () => p.checkAuth();
+  return service;
+}
+function createJudgeServiceFromEnv(env) {
+  const level = resolveLogLevel(env);
+  return createJudgeService(
+    resolveJudgeServiceEnv(env),
+    void 0,
+    level === void 0 ? void 0 : new ConsoleLogger(level)
+  );
+}
+
+// src/server.ts
+import { createServer } from "http";
+import {
+  JUDGE_ENV as JUDGE_ENV3,
+  JUDGE_ROUTES
+} from "@unotest/protocol";
+var MAX_BODY_BYTES = 1024 * 1024;
+var DRAIN_TIMEOUT_MS = 5e3;
+function startJudgeServer(service, env, deps = {}) {
+  const ctx = {
+    service,
+    env,
+    logger: deps.logger ?? SILENT_LOGGER,
+    probe: deps.probe ?? createAuthProbe(service),
+    now: deps.nowFn ?? Date.now
+  };
+  const server = createServer((req, res) => {
+    void route(ctx, req, res);
+  });
+  return new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(env.port, env.host, () => {
+      const addr = server.address();
+      const port = typeof addr === "object" && addr !== null ? addr.port : env.port;
+      resolve({
+        server,
+        port,
+        close: () => new Promise((res2, rej2) => server.close((e) => e ? rej2(e) : res2()))
+      });
+    });
+  });
+}
+async function route(ctx, req, res) {
+  const url = req.url ?? "/";
+  if (req.method === "GET" && url === JUDGE_ROUTES.health) {
+    const health = await ctx.probe.status();
+    sendJson(res, health.ok ? 200 : 503, health);
+    return;
+  }
+  if (req.method !== "POST" || url !== JUDGE_ROUTES.judge) {
+    sendError(res, 404, `unknown route ${req.method} ${url}`, "bad-request");
+    return;
+  }
+  if (ctx.env.token && req.headers.authorization !== `Bearer ${ctx.env.token}`) {
+    sendError(res, 401, `missing or wrong bearer token (${JUDGE_ENV3.token})`, "unauthorized");
+    return;
+  }
+  let body;
+  try {
+    body = parseRequest(await readBody(req, res));
+  } catch (e) {
+    if (res.writableEnded) return;
+    const message = errorMessage(e);
+    ctx.logger.error(`POST /judge  ERROR bad-request  ${message}`);
+    sendError(res, 400, message, "bad-request");
+    return;
+  }
+  const started = ctx.now();
+  try {
+    const verdict = await ctx.service.judge(body);
+    ctx.logger.info(verdictLine(verdict, ctx.now() - started));
+    sendJson(res, 200, verdict);
+  } catch (e) {
+    const code = judgeErrorCode(e);
+    ctx.logger.error(`POST /judge  ERROR ${code}  ${errorMessage(e)}`);
+    sendError(res, code === "provider-error" ? 502 : 500, errorMessage(e), code);
+  }
+}
+function verdictLine(verdict, ms) {
+  return `POST /judge  ${verdict.verdict}  attempts ${verdict.attempts}  ${verdict.model}  ${ms}ms`;
+}
+function parseRequest(raw) {
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error("request body is not JSON");
+  }
+  const { rubric, text } = parsed;
+  if (typeof rubric !== "string" || rubric.trim() === "") {
+    throw new Error('request body needs a non-empty string "rubric"');
+  }
+  if (typeof text !== "string") {
+    throw new Error('request body needs a string "text"');
+  }
+  return { rubric, text };
+}
+function readBody(req, res) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let size = 0;
+    let overflowed = false;
+    req.on("data", (chunk) => {
+      if (overflowed) return;
+      size += chunk.length;
+      if (size > MAX_BODY_BYTES) {
+        overflowed = true;
+        sendError(
+          res,
+          413,
+          `request body exceeds ${MAX_BODY_BYTES} bytes \u2014 judge a narrower locator, not the whole page`,
+          "bad-request"
+        );
+        drain(req);
+        reject(new Error("request body too large"));
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+    req.on("error", reject);
+  });
+}
+function drain(req) {
+  const timer = setTimeout(() => req.destroy(), DRAIN_TIMEOUT_MS);
+  timer.unref();
+  req.on("end", () => clearTimeout(timer));
+  req.on("error", () => clearTimeout(timer));
+  req.resume();
+}
+function sendJson(res, status, body) {
+  const payload = JSON.stringify(body);
+  res.writeHead(status, { "content-type": "application/json" });
+  res.end(payload);
+}
+function sendError(res, status, error, code) {
+  sendJson(res, status, { error, code });
+}
+
+export {
+  JudgeError,
+  JudgeConfigError,
+  JudgeProviderError,
+  JudgeTransientError,
+  judgeErrorCode,
+  FAKE_MODEL_ID,
+  parseFakeRubric,
+  FakeJudgeProvider,
+  VERDICT_PROMPT,
+  parseVerdictReply,
+  vertexEndpoint,
+  VertexJudgeProvider,
+  GeminiJudgeProvider,
+  OpenAiJudgeProvider,
+  AnthropicJudgeProvider,
+  runProcess,
+  ClaudeCliJudgeProvider,
+  judgeWithRetries,
+  judgeWithVote,
+  LOG_LEVELS,
+  ConsoleLogger,
+  SILENT_LOGGER,
+  LoggingJudgeProvider,
+  withLogging,
+  TransientRetryProvider,
+  withTransientRetry,
+  createAuthProbe,
+  AuthProbe,
+  resolveJudgeServiceEnv,
+  resolveJudgeServerEnv,
+  resolveLogLevel,
+  buildProvider,
+  createJudgeService,
+  createJudgeServiceFromEnv,
+  startJudgeServer
+};
+//# sourceMappingURL=chunk-ADU4QPCJ.js.map