@unotest/judge 0.23.0 → 0.25.0

package/dist/chunk-4VUHPR26.js

@@ -1,660 +0,0 @@
-// src/errors.ts
-var JudgeError = class extends Error {
-  constructor(message, context = {}) {
-    super(message);
-    this.context = context;
-    this.name = new.target.name;
-  }
-  context;
-};
-var JudgeConfigError = class extends JudgeError {
-};
-var JudgeProviderError = class extends JudgeError {
-};
-
-// src/providers/fake.ts
-var FAKE_MODEL_ID = "fake";
-var CONSTRAINT_RE = /^must(?<not>\s+not)?\s+contain:\s*(?<needle>.+)$/i;
-function parseFakeRubric(rubric) {
-  const constraints = [];
-  for (const line of rubric.split(/\r?\n/)) {
-    const m = CONSTRAINT_RE.exec(line.trim());
-    if (!m?.groups?.needle) continue;
-    constraints.push({
-      kind: m.groups.not ? "not-contains" : "contains",
-      needle: m.groups.needle.trim()
-    });
-  }
-  return constraints;
-}
-var FakeJudgeProvider = class {
-  async judgeOnce(request) {
-    const constraints = parseFakeRubric(request.rubric);
-    if (constraints.length === 0) {
-      return {
-        pass: false,
-        reasoning: 'fake provider: rubric has no parseable constraints \u2014 use lines like "must contain: <substring>" / "must not contain: <substring>", or switch to a real provider (JUDGE_PROVIDER=vertex|claude|gemini|openai|anthropic)',
-        model: FAKE_MODEL_ID
-      };
-    }
-    const text = request.text.toLowerCase();
-    const violations = [];
-    for (const c of constraints) {
-      const hit = text.includes(c.needle.toLowerCase());
-      if (c.kind === "contains" && !hit) violations.push(`missing required "${c.needle}"`);
-      if (c.kind === "not-contains" && hit) violations.push(`contains forbidden "${c.needle}"`);
-    }
-    if (violations.length > 0) {
-      return { pass: false, reasoning: violations.join("; "), model: FAKE_MODEL_ID };
-    }
-    return {
-      pass: true,
-      reasoning: `all ${constraints.length} constraint(s) satisfied`,
-      model: FAKE_MODEL_ID
-    };
-  }
-};
-
-// src/verdict.ts
-var VERDICT_PROMPT = (rubric, text) => `You are a strict test judge. Evaluate the TEXT against the RUBRIC.
-Reply with ONLY a raw JSON object, no markdown code fences: {"verdict": "pass" | "fail", "reasoning": "<one short sentence>"}.
-RUBRIC:
-${rubric}
-
-TEXT:
-${text}`;
-var FENCED_JSON = /^```(?:json)?\s*\n([\s\S]*?)\n\s*```$/;
-function parseVerdictReply(text, model) {
-  const trimmed = text.trim();
-  const raw = FENCED_JSON.exec(trimmed)?.[1] ?? trimmed;
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    throw new JudgeProviderError(
-      `model "${model}" did not reply with the requested JSON verdict: ${text.slice(0, 200)}`,
-      { model }
-    );
-  }
-  const { verdict, reasoning } = parsed;
-  if (verdict !== "pass" && verdict !== "fail") {
-    throw new JudgeProviderError(
-      `model "${model}" replied with an unknown verdict ${JSON.stringify(verdict)} (expected "pass"/"fail")`,
-      { model }
-    );
-  }
-  return {
-    pass: verdict === "pass",
-    reasoning: typeof reasoning === "string" && reasoning.length > 0 ? reasoning : "(no reasoning)"
-  };
-}
-
-// src/providers/generate-content.ts
-function generateContentBody(request) {
-  return {
-    contents: [
-      { role: "user", parts: [{ text: VERDICT_PROMPT(request.rubric, request.text) }] }
-    ],
-    generationConfig: {
-      temperature: 0,
-      responseMimeType: "application/json"
-    }
-  };
-}
-function extractCandidateText(body, model, backend) {
-  let parsed;
-  try {
-    parsed = JSON.parse(body);
-  } catch {
-    throw new JudgeProviderError(`${backend} reply is not JSON: ${body.slice(0, 200)}`, { model });
-  }
-  const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
-  if (typeof text !== "string" || text.length === 0) {
-    throw new JudgeProviderError(
-      `${backend} reply carries no candidate text (blocked or empty): ${body.slice(0, 300)}`,
-      { model }
-    );
-  }
-  return text;
-}
-
-// src/providers/http-json.ts
-async function postJson(call) {
-  const { url, headers, body, timeoutMs, backend, model, fetchImpl } = call;
-  let res;
-  try {
-    res = await fetchImpl(url, {
-      method: "POST",
-      headers: { "content-type": "application/json", ...headers },
-      body: JSON.stringify(body),
-      signal: AbortSignal.timeout(timeoutMs)
-    });
-  } catch (e) {
-    throw new JudgeProviderError(
-      `cannot reach ${backend} at ${url} (${e instanceof Error ? e.message : String(e)})`,
-      { url }
-    );
-  }
-  const text = await res.text();
-  if (!res.ok) {
-    throw new JudgeProviderError(
-      `${backend} replied ${res.status} for model "${model}": ${text.slice(0, 500)}`,
-      { status: res.status, model }
-    );
-  }
-  return text;
-}
-
-// src/providers/vertex.ts
-var VertexJudgeProvider = class {
-  constructor(opts) {
-    this.opts = opts;
-    this.fetchImpl = opts.fetchImpl ?? fetch;
-  }
-  opts;
-  fetchImpl;
-  tokenSource;
-  async judgeOnce(request) {
-    const { project, location, model, timeoutMs } = this.opts;
-    const url = `https://${location}-aiplatform.googleapis.com/v1/projects/${project}/locations/${location}/publishers/google/models/${model}:generateContent`;
-    const token = await this.accessToken();
-    const body = await postJson({
-      url,
-      headers: { authorization: `Bearer ${token}` },
-      body: generateContentBody(request),
-      timeoutMs,
-      backend: "Vertex AI",
-      model,
-      fetchImpl: this.fetchImpl
-    });
-    return { ...parseVerdictReply(extractCandidateText(body, model, "Vertex AI"), model), model };
-  }
-  async accessToken() {
-    if (this.opts.accessToken) return this.opts.accessToken;
-    this.tokenSource ??= await buildAdcTokenSource();
-    return this.tokenSource();
-  }
-};
-async function buildAdcTokenSource() {
-  const specifier = "google-auth-library";
-  let mod;
-  try {
-    mod = await import(specifier);
-  } catch {
-    throw new JudgeConfigError(
-      'Vertex provider needs Application Default Credentials \u2014 install the optional peer "google-auth-library" (`npm i google-auth-library`), or pass a token via JUDGE_ACCESS_TOKEN'
-    );
-  }
-  const auth = new mod.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/cloud-platform"] });
-  return async () => {
-    const token = await auth.getAccessToken();
-    if (!token) {
-      throw new JudgeConfigError(
-        "ADC produced no access token \u2014 run `gcloud auth application-default login` or set GOOGLE_APPLICATION_CREDENTIALS"
-      );
-    }
-    return token;
-  };
-}
-
-// src/providers/gemini.ts
-var GeminiJudgeProvider = class {
-  constructor(opts) {
-    this.opts = opts;
-    this.fetchImpl = opts.fetchImpl ?? fetch;
-  }
-  opts;
-  fetchImpl;
-  async judgeOnce(request) {
-    const { apiKey, model, timeoutMs } = this.opts;
-    const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`;
-    const body = await postJson({
-      url,
-      headers: { "x-goog-api-key": apiKey },
-      body: generateContentBody(request),
-      timeoutMs,
-      backend: "Gemini API",
-      model,
-      fetchImpl: this.fetchImpl
-    });
-    return { ...parseVerdictReply(extractCandidateText(body, model, "Gemini API"), model), model };
-  }
-};
-
-// src/providers/openai.ts
-var OpenAiJudgeProvider = class {
-  constructor(opts) {
-    this.opts = opts;
-    this.fetchImpl = opts.fetchImpl ?? fetch;
-  }
-  opts;
-  fetchImpl;
-  async judgeOnce(request) {
-    const { apiKey, model, timeoutMs } = this.opts;
-    const body = await postJson({
-      url: "https://api.openai.com/v1/chat/completions",
-      headers: { authorization: `Bearer ${apiKey}` },
-      body: {
-        model,
-        messages: [{ role: "user", content: VERDICT_PROMPT(request.rubric, request.text) }],
-        response_format: { type: "json_object" }
-      },
-      timeoutMs,
-      backend: "OpenAI",
-      model,
-      fetchImpl: this.fetchImpl
-    });
-    return { ...parseVerdictReply(extractMessageContent(body, model), model), model };
-  }
-};
-function extractMessageContent(body, model) {
-  let parsed;
-  try {
-    parsed = JSON.parse(body);
-  } catch {
-    throw new JudgeProviderError(`OpenAI reply is not JSON: ${body.slice(0, 200)}`, { model });
-  }
-  const content = parsed.choices?.[0]?.message?.content;
-  if (typeof content !== "string" || content.length === 0) {
-    throw new JudgeProviderError(
-      `OpenAI reply carries no message content: ${body.slice(0, 300)}`,
-      { model }
-    );
-  }
-  return content;
-}
-
-// src/providers/anthropic.ts
-var ANTHROPIC_VERSION = "2023-06-01";
-var MAX_TOKENS = 1024;
-var AnthropicJudgeProvider = class {
-  constructor(opts) {
-    this.opts = opts;
-    this.fetchImpl = opts.fetchImpl ?? fetch;
-  }
-  opts;
-  fetchImpl;
-  async judgeOnce(request) {
-    const { apiKey, model, timeoutMs } = this.opts;
-    const body = await postJson({
-      url: "https://api.anthropic.com/v1/messages",
-      headers: { "x-api-key": apiKey, "anthropic-version": ANTHROPIC_VERSION },
-      body: {
-        model,
-        max_tokens: MAX_TOKENS,
-        messages: [{ role: "user", content: VERDICT_PROMPT(request.rubric, request.text) }]
-      },
-      timeoutMs,
-      backend: "Anthropic API",
-      model,
-      fetchImpl: this.fetchImpl
-    });
-    return { ...parseVerdictReply(extractTextBlock(body, model), model), model };
-  }
-};
-function extractTextBlock(body, model) {
-  let parsed;
-  try {
-    parsed = JSON.parse(body);
-  } catch {
-    throw new JudgeProviderError(`Anthropic API reply is not JSON: ${body.slice(0, 200)}`, {
-      model
-    });
-  }
-  const blocks = parsed.content;
-  const text = blocks?.find((b) => b.type === "text")?.text;
-  if (typeof text !== "string" || text.length === 0) {
-    throw new JudgeProviderError(
-      `Anthropic API reply carries no text block (refusal or empty): ${body.slice(0, 300)}`,
-      { model }
-    );
-  }
-  return text;
-}
-
-// src/providers/claude-cli.ts
-var MAX_STDOUT_BYTES = 10 * 1024 * 1024;
-var ClaudeCliJudgeProvider = class {
-  constructor(opts) {
-    this.opts = opts;
-  }
-  opts;
-  async judgeOnce(request) {
-    const { bin, model, timeoutMs } = this.opts;
-    const args = [
-      "-p",
-      "--output-format",
-      "json",
-      "--strict-mcp-config",
-      ...model ? ["--model", model] : [],
-      VERDICT_PROMPT(request.rubric, request.text)
-    ];
-    const exec = this.opts.execImpl ?? await defaultExec();
-    let stdout;
-    try {
-      ({ stdout } = await exec(bin, args, { timeout: timeoutMs, maxBuffer: MAX_STDOUT_BYTES }));
-    } catch (e) {
-      throw toTypedError(e, bin, timeoutMs);
-    }
-    const envelope = parseEnvelope(stdout, bin);
-    const modelLabel = firstModelId(envelope) ?? model ?? "claude";
-    if (envelope.is_error === true || typeof envelope.result !== "string" || !envelope.result) {
-      throw new JudgeProviderError(
-        `${bin} -p returned an error result: ${String(envelope.result ?? stdout).slice(0, 300)}`,
-        { model: modelLabel }
-      );
-    }
-    return { ...parseVerdictReply(envelope.result, modelLabel), model: modelLabel };
-  }
-};
-async function defaultExec() {
-  const { execFile } = await import("child_process");
-  const { promisify } = await import("util");
-  return promisify(execFile);
-}
-function toTypedError(e, bin, timeoutMs) {
-  const err = e;
-  if (err.code === "ENOENT") {
-    return new JudgeConfigError(
-      `Claude Code CLI not found ("${bin}") \u2014 install it (https://claude.com/claude-code) or point JUDGE_CLAUDE_BIN at the binary`,
-      { bin }
-    );
-  }
-  if (err.code === "EACCES") {
-    return new JudgeConfigError(`Claude Code CLI is not executable ("${bin}")`, { bin });
-  }
-  if (err.killed === true) {
-    return new JudgeProviderError(`${bin} -p timed out after ${timeoutMs}ms`, { bin, timeoutMs });
-  }
-  const stderr = typeof err.stderr === "string" && err.stderr ? err.stderr : String(err.message ?? e);
-  return new JudgeProviderError(`${bin} -p failed: ${stderr.slice(0, 500)}`, { bin });
-}
-function parseEnvelope(stdout, bin) {
-  try {
-    return JSON.parse(stdout);
-  } catch {
-    throw new JudgeProviderError(
-      `${bin} -p did not return the JSON envelope: ${stdout.slice(0, 300)}`,
-      { bin }
-    );
-  }
-}
-function firstModelId(envelope) {
-  const keys = envelope.modelUsage ? Object.keys(envelope.modelUsage) : [];
-  return keys.length > 0 ? keys[0] : void 0;
-}
-
-// src/policy.ts
-async function judgeWithRetries(provider, request, retries) {
-  const maxAttempts = Math.max(0, retries) + 1;
-  let last = null;
-  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-    const single = await provider.judgeOnce(request);
-    last = {
-      verdict: single.pass ? "pass" : "fail",
-      reasoning: single.reasoning,
-      model: single.model,
-      attempts: attempt
-    };
-    if (single.pass) return last;
-  }
-  return last;
-}
-
-// src/env.ts
-var PROVIDER_KINDS = [
-  "fake",
-  "vertex",
-  "claude",
-  "gemini",
-  "openai",
-  "anthropic"
-];
-var DEFAULT_MODEL = {
-  fake: void 0,
-  vertex: "gemini-2.5-flash",
-  gemini: "gemini-2.5-flash",
-  openai: "gpt-5-mini",
-  anthropic: "claude-haiku-4-5",
-  claude: void 0
-};
-var API_KEY_ENV = {
-  gemini: "GEMINI_API_KEY",
-  openai: "OPENAI_API_KEY",
-  anthropic: "ANTHROPIC_API_KEY"
-};
-var DEFAULT_RETRIES = 1;
-var DEFAULT_TIMEOUT_MS = 3e4;
-var DEFAULT_CLAUDE_TIMEOUT_MS = 12e4;
-function intEnv(env, name, fallback, min, max) {
-  const raw = env[name];
-  if (raw === void 0 || raw === "") return fallback;
-  const n = Number.parseInt(raw, 10);
-  if (Number.isNaN(n) || n < min || n > max) {
-    throw new JudgeConfigError(`${name} must be an integer in [${min}, ${max}], got "${raw}"`);
-  }
-  return n;
-}
-function resolveJudgeServiceEnv(env) {
-  const provider = env.JUDGE_PROVIDER;
-  if (!provider || !PROVIDER_KINDS.includes(provider)) {
-    const list = PROVIDER_KINDS.map((k) => `"${k}"`).join(" | ");
-    throw new JudgeConfigError(
-      provider === void 0 || provider === "" ? `JUDGE_PROVIDER is not set \u2014 one of ${list} ("fake" is deterministic and CI-safe)` : `JUDGE_PROVIDER must be one of ${list}, got "${provider}"`
-    );
-  }
-  const resolved = {
-    provider,
-    retries: intEnv(env, "JUDGE_RETRIES", DEFAULT_RETRIES, 0, 10),
-    timeoutMs: intEnv(
-      env,
-      "JUDGE_TIMEOUT_MS",
-      provider === "claude" ? DEFAULT_CLAUDE_TIMEOUT_MS : DEFAULT_TIMEOUT_MS,
-      1e3,
-      6e5
-    )
-  };
-  const model = env.JUDGE_MODEL || DEFAULT_MODEL[provider];
-  if (model) resolved.model = model;
-  if (provider === "vertex") {
-    const project = env.GOOGLE_CLOUD_PROJECT;
-    const location = env.GOOGLE_CLOUD_LOCATION;
-    if (!project || !location) {
-      throw new JudgeConfigError(
-        "JUDGE_PROVIDER=vertex requires GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION (ADC auth, no API keys)"
-      );
-    }
-    resolved.project = project;
-    resolved.location = location;
-    if (env.JUDGE_ACCESS_TOKEN) resolved.accessToken = env.JUDGE_ACCESS_TOKEN;
-  }
-  const keyEnvName = API_KEY_ENV[provider];
-  if (keyEnvName) {
-    const apiKey = env[keyEnvName];
-    if (!apiKey) {
-      throw new JudgeConfigError(`JUDGE_PROVIDER=${provider} requires ${keyEnvName}`);
-    }
-    resolved.apiKey = apiKey;
-  }
-  if (provider === "claude") {
-    resolved.claudeBin = env.JUDGE_CLAUDE_BIN || "claude";
-  }
-  return resolved;
-}
-function resolveJudgeServerEnv(env) {
-  const resolved = {
-    host: env.JUDGE_HOST || "127.0.0.1",
-    port: intEnv(env, "JUDGE_PORT", 8790, 1, 65535)
-  };
-  if (env.JUDGE_TOKEN) resolved.token = env.JUDGE_TOKEN;
-  return resolved;
-}
-
-// src/judge-service.ts
-function buildProvider(env) {
-  switch (env.provider) {
-    case "fake":
-      return new FakeJudgeProvider();
-    case "vertex":
-      return new VertexJudgeProvider({
-        project: env.project,
-        location: env.location,
-        model: env.model,
-        timeoutMs: env.timeoutMs,
-        ...env.accessToken ? { accessToken: env.accessToken } : {}
-      });
-    case "gemini":
-      return new GeminiJudgeProvider({
-        apiKey: env.apiKey,
-        model: env.model,
-        timeoutMs: env.timeoutMs
-      });
-    case "openai":
-      return new OpenAiJudgeProvider({
-        apiKey: env.apiKey,
-        model: env.model,
-        timeoutMs: env.timeoutMs
-      });
-    case "anthropic":
-      return new AnthropicJudgeProvider({
-        apiKey: env.apiKey,
-        model: env.model,
-        timeoutMs: env.timeoutMs
-      });
-    case "claude":
-      return new ClaudeCliJudgeProvider({
-        bin: env.claudeBin,
-        timeoutMs: env.timeoutMs,
-        ...env.model ? { model: env.model } : {}
-      });
-  }
-}
-function createJudgeService(env, provider) {
-  const p = provider ?? buildProvider(env);
-  return {
-    judge: (request) => judgeWithRetries(p, request, env.retries)
-  };
-}
-function createJudgeServiceFromEnv(env) {
-  return createJudgeService(resolveJudgeServiceEnv(env));
-}
-
-// src/server.ts
-import { createServer } from "http";
-import { JUDGE_ROUTES } from "@unotest/protocol";
-var MAX_BODY_BYTES = 1024 * 1024;
-function startJudgeServer(service, env) {
-  const server = createServer((req, res) => {
-    void route(service, env, req, res);
-  });
-  return new Promise((resolve, reject) => {
-    server.once("error", reject);
-    server.listen(env.port, env.host, () => {
-      const addr = server.address();
-      const port = typeof addr === "object" && addr !== null ? addr.port : env.port;
-      resolve({
-        server,
-        port,
-        close: () => new Promise((res2, rej2) => server.close((e) => e ? rej2(e) : res2()))
-      });
-    });
-  });
-}
-async function route(service, env, req, res) {
-  const url = req.url ?? "/";
-  if (req.method === "GET" && url === JUDGE_ROUTES.health) {
-    sendJson(res, 200, { ok: true });
-    return;
-  }
-  if (req.method !== "POST" || url !== JUDGE_ROUTES.judge) {
-    sendError(res, 404, `unknown route ${req.method} ${url}`, "bad-request");
-    return;
-  }
-  if (env.token && req.headers.authorization !== `Bearer ${env.token}`) {
-    sendError(res, 401, "missing or wrong bearer token (JUDGE_TOKEN)", "unauthorized");
-    return;
-  }
-  let body;
-  try {
-    body = parseRequest(await readBody(req));
-  } catch (e) {
-    sendError(res, 400, e instanceof Error ? e.message : String(e), "bad-request");
-    return;
-  }
-  try {
-    sendJson(res, 200, await service.judge(body));
-  } catch (e) {
-    if (e instanceof JudgeProviderError) {
-      sendError(res, 502, e.message, "provider-error");
-    } else if (e instanceof JudgeConfigError) {
-      sendError(res, 500, e.message, "internal");
-    } else {
-      sendError(res, 500, e instanceof Error ? e.message : String(e), "internal");
-    }
-  }
-}
-function parseRequest(raw) {
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    throw new Error("request body is not JSON");
-  }
-  const { rubric, text } = parsed;
-  if (typeof rubric !== "string" || rubric.trim() === "") {
-    throw new Error('request body needs a non-empty string "rubric"');
-  }
-  if (typeof text !== "string") {
-    throw new Error('request body needs a string "text"');
-  }
-  return { rubric, text };
-}
-function readBody(req) {
-  return new Promise((resolve, reject) => {
-    const chunks = [];
-    let size = 0;
-    req.on("data", (chunk) => {
-      size += chunk.length;
-      if (size > MAX_BODY_BYTES) {
-        reject(new Error(`request body exceeds ${MAX_BODY_BYTES} bytes`));
-        req.destroy();
-        return;
-      }
-      chunks.push(chunk);
-    });
-    req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
-    req.on("error", reject);
-  });
-}
-function sendJson(res, status, body) {
-  const payload = JSON.stringify(body);
-  res.writeHead(status, { "content-type": "application/json" });
-  res.end(payload);
-}
-function sendError(res, status, error, code) {
-  sendJson(res, status, { error, code });
-}
-
-export {
-  JudgeError,
-  JudgeConfigError,
-  JudgeProviderError,
-  FAKE_MODEL_ID,
-  parseFakeRubric,
-  FakeJudgeProvider,
-  VERDICT_PROMPT,
-  parseVerdictReply,
-  VertexJudgeProvider,
-  GeminiJudgeProvider,
-  OpenAiJudgeProvider,
-  AnthropicJudgeProvider,
-  ClaudeCliJudgeProvider,
-  judgeWithRetries,
-  resolveJudgeServiceEnv,
-  resolveJudgeServerEnv,
-  buildProvider,
-  createJudgeService,
-  createJudgeServiceFromEnv,
-  startJudgeServer
-};
-//# sourceMappingURL=chunk-4VUHPR26.js.map