@unotest/judge 0.24.0 → 0.25.0

package/dist/cli.js

@@ -1,11 +1,15 @@
 import {
+  ConsoleLogger,
+  createAuthProbe,
   createJudgeService,
   resolveJudgeServerEnv,
   resolveJudgeServiceEnv,
+  resolveLogLevel,
   startJudgeServer
-} from "./chunk-74FL5RWE.js";
+} from "./chunk-ADU4QPCJ.js";
 
 // src/cli.ts
+import { JUDGE_ENV } from "@unotest/protocol";
 async function main(argv = process.argv.slice(2)) {
   if (argv.includes("--help") || argv.includes("-h")) {
     printHelp();
@@ -21,12 +25,46 @@ async function main(argv = process.argv.slice(2)) {
   }
   const serviceEnv = resolveJudgeServiceEnv(process.env);
   const serverEnv = resolveJudgeServerEnv(process.env);
-  const started = await startJudgeServer(createJudgeService(serviceEnv), serverEnv);
-  console.log(
+  const logger = new ConsoleLogger(resolveLogLevel(process.env) ?? "info");
+  if (serviceEnv.votes > 1 && process.env[JUDGE_ENV.retries]) {
+    logger.warn(
+      `${JUDGE_ENV.retries} is ignored while ${JUDGE_ENV.vote}=${serviceEnv.votes}: re-asking on fail biases toward pass, which is what a vote exists to remove`
+    );
+  }
+  const service = createJudgeService(serviceEnv, void 0, logger);
+  const probe = createAuthProbe(service);
+  const health = await probe.status(true);
+  const reason = health.error ?? "the provider rejected the credentials";
+  if (argv.includes("--check")) {
+    if (health.ok) {
+      logger.info(`\u2713 ${serviceEnv.provider} credentials accepted`);
+      return;
+    }
+    logger.error(`\u2717 ${reason}`);
+    process.exitCode = 1;
+    return;
+  }
+  if (!health.ok) {
+    if (!serverEnv.skipPreflight) {
+      logger.error(`\u2717 ${reason}`);
+      logger.error(
+        `  (set ${JUDGE_ENV.skipPreflight}=1 to start anyway \u2014 the first verdict will fail instead, and /health will keep reporting this)`
+      );
+      process.exitCode = 1;
+      return;
+    }
+    logger.warn(`! starting with ${JUDGE_ENV.skipPreflight}=1 despite: ${reason}`);
+  }
+  const started = await startJudgeServer(service, serverEnv, { logger, probe });
+  logger.info(
     `unotest-judge listening on http://${serverEnv.host}:${started.port} (provider: ${serviceEnv.provider}${serviceEnv.model ? `, model: ${serviceEnv.model}` : ""})`
   );
+  installShutdown(started.close, logger);
+}
+function installShutdown(close, logger) {
   const shutdown = () => {
-    void started.close().finally(() => process.exit(0));
+    logger.debug("unotest-judge shutting down");
+    void close().finally(() => process.exit(0));
   };
   process.once("SIGINT", shutdown);
   process.once("SIGTERM", shutdown);
@@ -36,7 +74,10 @@ function printHelp() {
     [
       "unotest-judge \u2014 LLM-judge service for @unotest/web's assertJudge",
       "",
-      "Usage: npx @unotest/judge [--help] [--version]",
+      "Usage: npx @unotest/judge [--check] [--help] [--version]",
+      "",
+      "  --check   verify the provider's credentials and exit (0 = usable),",
+      "            without listening. The same check runs on every start.",
       "",
       "Env:",
       "  UNOTEST_JUDGE_PROVIDER        fake (deterministic, CI-safe) | vertex (ADC) |",
@@ -50,8 +91,14 @@ function printHelp() {
       "  UNOTEST_JUDGE_VOTE            N-call majority vote (default 1 = off)",
       "  UNOTEST_JUDGE_CALL_TIMEOUT_MS per-call budget, ms (default 30000;",
       "                                claude 120000)",
+      "  UNOTEST_JUDGE_LOG_LEVEL       silent | error | warn | info (default) | debug;",
+      "                                falls back to UNOTEST_LOG_LEVEL. debug prints the",
+      "                                effective prompt, the judged text and raw replies",
+      "                                \u2014 application content, so opt-in only",
+      "  UNOTEST_JUDGE_SKIP_PREFLIGHT  1 = start even when the credential check fails",
+      "                                (/health keeps reporting the failure)",
       "  GOOGLE_CLOUD_PROJECT          vertex: ADC project",
-      "  GOOGLE_CLOUD_LOCATION         vertex: ADC location",
+      "  GOOGLE_CLOUD_LOCATION         vertex: global, or a region (us-central1)",
       "  UNOTEST_JUDGE_ACCESS_TOKEN    vertex: static bearer override (skips ADC)",
       "  GEMINI_API_KEY                gemini: API key",
       "  OPENAI_API_KEY                openai: API key",

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { JudgeRequest, JudgeVerdict } from '@unotest/protocol';
-export { JUDGE_ROUTES, JudgeErrorResponse, JudgeRequest, JudgeVerdict } from '@unotest/protocol';
+import { JudgeErrorResponse, JudgeRequest, JudgeVerdict, JudgeHealthResponse } from '@unotest/protocol';
+export { JUDGE_ROUTES, JudgeErrorResponse, JudgeHealthResponse, JudgeRequest, JudgeVerdict } from '@unotest/protocol';
 import { Server } from 'node:http';
 
 declare class JudgeError extends Error {
@@ -13,17 +13,41 @@ declare class JudgeConfigError extends JudgeError {
  *  model backend, or an unparseable model reply. */
 declare class JudgeProviderError extends JudgeError {
 }
+/** A provider fault that is expected to pass on its own: rate limit,
+ *  backend 5xx, connection reset. Separate class so the transport layer can
+ *  re-send it — a verdict is never retried this way (that is policy.ts's
+ *  job and a deliberate bias), only the call that never produced one. */
+declare class JudgeTransientError extends JudgeProviderError {
+}
+/** Wire code for an error — one mapping, used by the HTTP route and by the
+ *  health probe so a fault reads the same whether it arrives as a failed
+ *  verdict or as a red /health. */
+declare function judgeErrorCode(e: unknown): NonNullable<JudgeErrorResponse["code"]>;
 
 interface SingleVerdict {
     pass: boolean;
     reasoning: string;
     /** Model id that produced this verdict (`fake` for the fake provider). */
     model: string;
+    /** The backend's reply as received, before parsing. Carried for debug
+     *  logging only — set by the verdict parser, so every model-backed
+     *  provider gets it without plumbing of its own. */
+    raw?: string;
 }
 interface JudgeProvider {
     /** One un-retried judgement. Throws JudgeProviderError on transport /
      *  parse faults; a clean `fail` verdict is a RESULT, not an error. */
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    /** Prove the credentials work WITHOUT calling the model — a token
+     *  exchange, a free `GET /models`, a `--version` spawn. Used by the
+     *  startup preflight and by /health, so a judge that cannot judge says so
+     *  before a scenario spends five minutes discovering it.
+     *
+     *  Optional by design (ISP): the fake provider has no credentials, and a
+     *  future provider without a free probe stays honest by omitting it
+     *  rather than faking a green check. Throws the same typed errors as
+     *  `judgeOnce` — JudgeConfigError when the operator must act. */
+    checkAuth?(): Promise<void>;
 }
 
 declare const FAKE_MODEL_ID = "fake";
@@ -47,12 +71,20 @@ interface VertexProviderOptions {
     /** Injected fetch (tests). Defaults to global fetch. */
     fetchImpl?: typeof fetch;
 }
+/** Exported for unit tests — the URL shape is the whole bug surface here. */
+declare function vertexEndpoint(project: string, location: string, model: string): string;
 declare class VertexJudgeProvider implements JudgeProvider {
     private readonly opts;
-    private readonly fetchImpl;
+    private readonly api;
     private tokenSource;
     constructor(opts: VertexProviderOptions);
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    /** Two credentials, two free probes. ADC: fetching a token exercises the
+     *  exact thing that dies on an org's daily reauth policy. A static token:
+     *  Google's own `tokeninfo` says whether it is still alive — and a static
+     *  token lives about an hour, so "valid when the service started" is not
+     *  the same question as "valid now". */
+    checkAuth(): Promise<void>;
     private accessToken;
 }
 
@@ -66,9 +98,10 @@ interface GeminiProviderOptions {
 }
 declare class GeminiJudgeProvider implements JudgeProvider {
     private readonly opts;
-    private readonly fetchImpl;
+    private readonly api;
     constructor(opts: GeminiProviderOptions);
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    checkAuth(): Promise<void>;
 }
 
 interface OpenAiProviderOptions {
@@ -81,9 +114,11 @@ interface OpenAiProviderOptions {
 }
 declare class OpenAiJudgeProvider implements JudgeProvider {
     private readonly opts;
-    private readonly fetchImpl;
+    private readonly api;
     constructor(opts: OpenAiProviderOptions);
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    /** `GET /models` is free and needs the same key the verdict call uses. */
+    checkAuth(): Promise<void>;
 }
 
 interface AnthropicProviderOptions {
@@ -96,19 +131,31 @@ interface AnthropicProviderOptions {
 }
 declare class AnthropicJudgeProvider implements JudgeProvider {
     private readonly opts;
-    private readonly fetchImpl;
+    private readonly api;
     constructor(opts: AnthropicProviderOptions);
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    /** `GET /models` is free and needs the same key the verdict call uses. */
+    checkAuth(): Promise<void>;
 }
 
-interface ExecFileResult {
+interface ProcessResult {
     stdout: string;
     stderr: string;
 }
-type ExecFileFn = (file: string, args: string[], opts: {
-    timeout: number;
+interface RunProcessOptions {
+    /** Wall-clock budget; the child is killed past it. */
+    timeoutMs: number;
+    /** Cap on captured stdout — a runaway child must not eat the service. */
     maxBuffer: number;
-}) => Promise<ExecFileResult>;
+    /** Written to the child's stdin, which is then closed. */
+    input?: string;
+}
+type RunProcess = (file: string, args: string[], opts: RunProcessOptions) => Promise<ProcessResult>;
+/** Rejects with an Error carrying `code` / `killed` / `stderr`, the same
+ *  shape `child_process.execFile` uses — callers discriminate on those
+ *  fields, not on message text. */
+declare const runProcess: RunProcess;
+
 interface ClaudeCliProviderOptions {
     /** Binary to spawn (UNOTEST_JUDGE_CLAUDE_BIN, default "claude"). */
     bin: string;
@@ -116,23 +163,96 @@ interface ClaudeCliProviderOptions {
     model?: string;
     /** Per-call wall-clock budget, ms. */
     timeoutMs: number;
-    /** Injected exec (tests). Defaults to node:child_process execFile. */
-    execImpl?: ExecFileFn;
+    /** Injected process runner (tests). Defaults to a real spawn. */
+    runImpl?: RunProcess;
 }
 declare class ClaudeCliJudgeProvider implements JudgeProvider {
     private readonly opts;
+    private readonly run;
     constructor(opts: ClaudeCliProviderOptions);
     judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    /** The binary exists and runs. It cannot prove the session is still
+     *  authenticated without spending a model call, so that failure stays a
+     *  first-verdict one — but "claude is not installed on this box" is the
+     *  common case and it is caught here. */
+    checkAuth(): Promise<void>;
 }
 
 declare const VERDICT_PROMPT: (rubric: string, text: string) => string;
-/** Parse the model's JSON verdict. Exported for unit tests. */
+/** Parse the model's JSON verdict. Exported for unit tests. `raw` is carried
+ *  through for debug logging. */
 declare function parseVerdictReply(text: string, model: string): {
     pass: boolean;
     reasoning: string;
+    raw: string;
 };
 
 declare function judgeWithRetries(provider: JudgeProvider, request: JudgeRequest, retries: number): Promise<JudgeVerdict>;
+/** `votes` independent calls, majority verdict wins. Calls run CONCURRENTLY:
+ *  sequential voting would multiply latency by N and blow the client's
+ *  whole-request budget on a slow provider. `votes` is odd by construction
+ *  (env validation) — a tie has no honest answer.
+ *
+ *  A ballot that ERRORS is dropped, not fatal: N concurrent calls are N
+ *  chances to catch a rate limit, so `Promise.all` made the reliability
+ *  mechanism the least reliable configuration. The vote still has to mean
+ *  something, so a verdict needs a majority of the REQUESTED votes to have
+ *  been cast (2 of 3), and a tie among the cast ones — only reachable once
+ *  a ballot is missing — re-raises the failure instead of picking a side. */
+declare function judgeWithVote(provider: JudgeProvider, request: JudgeRequest, votes: number): Promise<JudgeVerdict>;
+
+type LogLevel = "silent" | "error" | "warn" | "info" | "debug";
+declare const LOG_LEVELS: readonly LogLevel[];
+interface Logger {
+    error(message: string): void;
+    warn(message: string): void;
+    info(message: string): void;
+    debug(message: string): void;
+}
+declare class ConsoleLogger implements Logger {
+    private readonly level;
+    private readonly out;
+    private readonly err;
+    constructor(level: LogLevel, out?: (line: string) => void, err?: (line: string) => void);
+    error(message: string): void;
+    warn(message: string): void;
+    info(message: string): void;
+    debug(message: string): void;
+    private enabled;
+}
+/** Default for in-process use (@unotest/web's local mode): the consumer owns
+ *  its own console, the engine must not write to it uninvited. */
+declare const SILENT_LOGGER: Logger;
+
+declare class LoggingJudgeProvider implements JudgeProvider {
+    private readonly inner;
+    private readonly logger;
+    private readonly nowFn;
+    /** Mirrors the inner provider's capability instead of faking a probe the
+     *  wrapped one does not have. */
+    readonly checkAuth?: () => Promise<void>;
+    constructor(inner: JudgeProvider, logger: Logger, nowFn?: () => number);
+    judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+}
+/** Wrap only when someone is listening — an undecorated provider keeps the
+ *  stack (and the stack traces) shorter in the common case. */
+declare function withLogging(provider: JudgeProvider, logger: Logger | undefined): JudgeProvider;
+
+type Sleep = (ms: number) => Promise<void>;
+interface TransientRetryOptions {
+    /** Injected sleep (tests) — keeps the backoff out of the test clock. */
+    sleepImpl?: Sleep;
+}
+declare class TransientRetryProvider implements JudgeProvider {
+    private readonly inner;
+    private readonly opts;
+    /** Mirrors the inner provider's capability rather than inventing one. */
+    readonly checkAuth?: () => Promise<void>;
+    constructor(inner: JudgeProvider, opts?: TransientRetryOptions);
+    judgeOnce(request: JudgeRequest): Promise<SingleVerdict>;
+    private retry;
+}
+declare function withTransientRetry(provider: JudgeProvider, opts?: TransientRetryOptions): JudgeProvider;
 
 type JudgeProviderKind = "fake" | "vertex" | "claude" | "gemini" | "openai" | "anthropic";
 interface JudgeServiceEnv {
@@ -163,24 +283,68 @@ interface JudgeServerEnv {
     port: number;
     /** Optional bearer token the server requires on every /judge call. */
     token?: string;
+    /** Boot without probing the provider's credentials, and answer /health
+     *  without probing them either. */
+    skipPreflight: boolean;
 }
 declare function resolveJudgeServerEnv(env: NodeJS.ProcessEnv): JudgeServerEnv;
+/** Log level, or `undefined` when neither variable is set — the caller picks
+ *  the default, and the two callers differ: the CLI service talks (info),
+ *  the in-process engine stays quiet. */
+declare function resolveLogLevel(env: NodeJS.ProcessEnv): LogLevel | undefined;
 
 interface JudgeService {
     judge(request: JudgeRequest): Promise<JudgeVerdict>;
+    /** The provider's free credential probe, forwarded so the CLI preflight
+     *  and /health depend on this facade rather than on a concrete provider.
+     *  Absent when the provider has none (fake). */
+    checkAuth?(): Promise<void>;
 }
 declare function buildProvider(env: JudgeServiceEnv): JudgeProvider;
-declare function createJudgeService(env: JudgeServiceEnv, provider?: JudgeProvider): JudgeService;
+declare function createJudgeService(env: JudgeServiceEnv, provider?: JudgeProvider, logger?: Logger): JudgeService;
 /** Everything from process-env in one call — the entry point @unotest/web's
  *  local mode uses. Throws JudgeConfigError with an actionable message on
- *  missing/malformed env. */
+ *  missing/malformed env. In-process callers own their console, so logging
+ *  stays silent unless the level was asked for explicitly. */
 declare function createJudgeServiceFromEnv(env: NodeJS.ProcessEnv): JudgeService;
 
+interface AuthProbeOptions {
+    /** The provider's probe. Absent = the provider has no free check. */
+    checkAuth?: () => Promise<void>;
+    ttlMs?: number;
+    /** Override the clock for deterministic tests. */
+    nowFn?: () => number;
+}
+/** The one place that binds a service to its probe. Note what it does NOT
+ *  take: a "skip" flag. UNOTEST_JUDGE_SKIP_PREFLIGHT waives the STARTUP
+ *  GATE, not the truth — a waived service still answers /health from a real
+ *  probe. The alternative was tried and is worse: our own message for a
+ *  failed preflight suggests that flag, so the operator who sets it is
+ *  precisely the one whose credentials are dead, and a green /health then
+ *  hides exactly the fault the flag was reached for. */
+declare function createAuthProbe(service: Pick<JudgeService, "checkAuth">): AuthProbe;
+declare class AuthProbe {
+    private readonly opts;
+    private cached;
+    constructor(opts: AuthProbeOptions);
+    /** @param fresh bypass the cache (startup preflight, `--check`). */
+    status(fresh?: boolean): Promise<JudgeHealthResponse>;
+    private run;
+}
+
 interface StartedJudgeServer {
     server: Server;
     port: number;
     close(): Promise<void>;
 }
-declare function startJudgeServer(service: JudgeService, env: JudgeServerEnv): Promise<StartedJudgeServer>;
+interface JudgeServerDeps {
+    logger?: Logger;
+    /** Shared with the CLI preflight so a boot check and a /health poll do not
+     *  probe the provider twice. */
+    probe?: AuthProbe;
+    /** Override the clock for deterministic tests. */
+    nowFn?: () => number;
+}
+declare function startJudgeServer(service: JudgeService, env: JudgeServerEnv, deps?: JudgeServerDeps): Promise<StartedJudgeServer>;
 
-export { AnthropicJudgeProvider, ClaudeCliJudgeProvider, type ExecFileFn, type ExecFileResult, FAKE_MODEL_ID, FakeJudgeProvider, GeminiJudgeProvider, JudgeConfigError, JudgeError, type JudgeProvider, JudgeProviderError, type JudgeProviderKind, type JudgeServerEnv, type JudgeService, type JudgeServiceEnv, OpenAiJudgeProvider, type SingleVerdict, type StartedJudgeServer, VERDICT_PROMPT, VertexJudgeProvider, buildProvider, createJudgeService, createJudgeServiceFromEnv, judgeWithRetries, parseFakeRubric, parseVerdictReply, resolveJudgeServerEnv, resolveJudgeServiceEnv, startJudgeServer };
+export { AnthropicJudgeProvider, AuthProbe, type AuthProbeOptions, ClaudeCliJudgeProvider, ConsoleLogger, FAKE_MODEL_ID, FakeJudgeProvider, GeminiJudgeProvider, JudgeConfigError, JudgeError, type JudgeProvider, JudgeProviderError, type JudgeProviderKind, type JudgeServerDeps, type JudgeServerEnv, type JudgeService, type JudgeServiceEnv, JudgeTransientError, LOG_LEVELS, type LogLevel, type Logger, LoggingJudgeProvider, OpenAiJudgeProvider, type ProcessResult, type RunProcess, type RunProcessOptions, SILENT_LOGGER, type SingleVerdict, type StartedJudgeServer, type TransientRetryOptions, TransientRetryProvider, VERDICT_PROMPT, VertexJudgeProvider, buildProvider, createAuthProbe, createJudgeService, createJudgeServiceFromEnv, judgeErrorCode, judgeWithRetries, judgeWithVote, parseFakeRubric, parseVerdictReply, resolveJudgeServerEnv, resolveJudgeServiceEnv, resolveLogLevel, runProcess, startJudgeServer, vertexEndpoint, withLogging, withTransientRetry };

package/dist/index.js

@@ -1,31 +1,48 @@
 import {
   AnthropicJudgeProvider,
+  AuthProbe,
   ClaudeCliJudgeProvider,
+  ConsoleLogger,
   FAKE_MODEL_ID,
   FakeJudgeProvider,
   GeminiJudgeProvider,
   JudgeConfigError,
   JudgeError,
   JudgeProviderError,
+  JudgeTransientError,
+  LOG_LEVELS,
+  LoggingJudgeProvider,
   OpenAiJudgeProvider,
+  SILENT_LOGGER,
+  TransientRetryProvider,
   VERDICT_PROMPT,
   VertexJudgeProvider,
   buildProvider,
+  createAuthProbe,
   createJudgeService,
   createJudgeServiceFromEnv,
+  judgeErrorCode,
   judgeWithRetries,
+  judgeWithVote,
   parseFakeRubric,
   parseVerdictReply,
   resolveJudgeServerEnv,
   resolveJudgeServiceEnv,
-  startJudgeServer
-} from "./chunk-74FL5RWE.js";
+  resolveLogLevel,
+  runProcess,
+  startJudgeServer,
+  vertexEndpoint,
+  withLogging,
+  withTransientRetry
+} from "./chunk-ADU4QPCJ.js";
 
 // src/index.ts
 import { JUDGE_ROUTES } from "@unotest/protocol";
 export {
   AnthropicJudgeProvider,
+  AuthProbe,
   ClaudeCliJudgeProvider,
+  ConsoleLogger,
   FAKE_MODEL_ID,
   FakeJudgeProvider,
   GeminiJudgeProvider,
@@ -33,17 +50,30 @@ export {
   JudgeConfigError,
   JudgeError,
   JudgeProviderError,
+  JudgeTransientError,
+  LOG_LEVELS,
+  LoggingJudgeProvider,
   OpenAiJudgeProvider,
+  SILENT_LOGGER,
+  TransientRetryProvider,
   VERDICT_PROMPT,
   VertexJudgeProvider,
   buildProvider,
+  createAuthProbe,
   createJudgeService,
   createJudgeServiceFromEnv,
+  judgeErrorCode,
   judgeWithRetries,
+  judgeWithVote,
   parseFakeRubric,
   parseVerdictReply,
   resolveJudgeServerEnv,
   resolveJudgeServiceEnv,
-  startJudgeServer
+  resolveLogLevel,
+  runProcess,
+  startJudgeServer,
+  vertexEndpoint,
+  withLogging,
+  withTransientRetry
 };
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unotest/judge",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "LLM-judge service for the @unotest ecosystem: judges free-form text (chat replies, generated content) against a natural-language rubric and returns a structured pass/fail verdict with reasoning. Runs as a small HTTP service (`npx @unotest/judge`) or in-process. Providers: deterministic `fake` (CI-safe), Google Vertex AI via ADC, the local Claude Code CLI (subscription auth, no API key), and Gemini / OpenAI / Anthropic APIs via keys — all raw HTTP or a local process, zero provider SDKs. Backs the `assertJudge` DSL assertion of @unotest/web.",
   "license": "MIT",
   "publishConfig": {
@@ -33,7 +33,7 @@
   ],
   "dependencies": {
     "google-auth-library": "^10.0.0",
-    "@unotest/protocol": "^0.24.0"
+    "@unotest/protocol": "^0.25.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.0",