@tangle-network/agent-eval 0.123.1 → 0.123.3

package/dist/{chunk-VJ7T5WIO.js → chunk-WP3GEXKI.js}

@@ -6,10 +6,10 @@ import {
   DEFAULT_TRACE_ANALYST_KINDS,
   createTraceAnalystKind,
   makeFinding
-} from "./chunk-FC5NDO3E.js";
+} from "./chunk-MHPEGJHC.js";
 import {
   LlmClient
-} from "./chunk-NJC7U437.js";
+} from "./chunk-MK3Q6GCL.js";
 import {
   spanEpochMillis
 } from "./chunk-IR3KBHOY.js";
@@ -547,4 +547,4 @@ export {
   behavioralAnalyst,
   buildDefaultAnalystRegistry
 };
-//# sourceMappingURL=chunk-VJ7T5WIO.js.map
+//# sourceMappingURL=chunk-WP3GEXKI.js.map

package/dist/cli.js

@@ -4,10 +4,10 @@ import {
   handleVersion,
   runRpcBatch,
   runRpcOnce,
-  startServer
-} from "./chunk-LBAHQOBI.js";
-import "./chunk-NJC7U437.js";
-import "./chunk-BGVTIE2C.js";
+  startServerAsync
+} from "./chunk-4KO6D2BO.js";
+import "./chunk-MK3Q6GCL.js";
+import "./chunk-GRCDRKII.js";
 import "./chunk-VI2UW6B6.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-ONWEPEDO.js";
@@ -15,6 +15,25 @@ import "./chunk-PZ5AY32C.js";
 
 // src/cli.ts
 import { writeFileSync } from "fs";
+
+// src/cli-config.ts
+function resolveCliLlmConfig(env = process.env) {
+  const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL);
+  const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY);
+  const openAiApiKey = nonEmpty(env.OPENAI_API_KEY);
+  const tangleApiKey = nonEmpty(env.TANGLE_API_KEY);
+  const baseUrl = explicitBaseUrl ?? nonEmpty(env.OPENAI_BASE_URL) ?? nonEmpty(env.TANGLE_ROUTER_URL) ?? (openAiApiKey ? "https://api.openai.com/v1" : void 0) ?? (tangleApiKey ? "https://router.tangle.tools/v1" : void 0);
+  const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey;
+  const model = nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL);
+  const client = baseUrl || apiKey ? { ...baseUrl ? { baseUrl } : {}, ...apiKey ? { apiKey } : {} } : void 0;
+  return { ...client ? { client } : {}, ...model ? { model } : {} };
+}
+function nonEmpty(value) {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : void 0;
+}
+
+// src/cli.ts
 function parseArgs(argv) {
   const [command, ...rest] = argv;
   const positional = [];
@@ -22,7 +41,17 @@ function parseArgs(argv) {
   for (let i = 0; i < rest.length; i++) {
     const tok = rest[i];
     if (tok.startsWith("--")) {
-      const key = tok.slice(2);
+      const raw = tok.slice(2);
+      const equalsAt = raw.indexOf("=");
+      if (equalsAt >= 0) {
+        flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1);
+        continue;
+      }
+      const key = raw;
+      if (key === "help") {
+        flags[key] = "true";
+        continue;
+      }
       const next = rest[i + 1];
       if (next != null && !next.startsWith("--")) {
         flags[key] = next;
@@ -30,20 +59,22 @@ function parseArgs(argv) {
       } else {
         flags[key] = "true";
       }
+    } else if (tok === "-h") {
+      flags.help = "true";
     } else {
       positional.push(tok);
     }
   }
   return { command: command ?? "help", positional, flags };
 }
-var HELP = `agent-eval \u2014 wire-protocol entry point.
+var HELP = `agent-eval: evaluation RPC and HTTP server.
 
 Commands:
   serve [--port 5005] [--host 127.0.0.1]
         Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.
   rpc <method>
         Read one JSON object from stdin (the params for <method>), write one
-        JSON object to stdout. Method \u2208 {judge, listRubrics, version}.
+        JSON object to stdout. Methods: judge, listRubrics, version.
   rpc-batch <method>
         Like 'rpc' but JSONL in / JSONL out.
   openapi [--out openapi.json]
@@ -51,14 +82,31 @@ Commands:
   version
         Print server + wire-protocol version JSON.
 
+Judge provider:
+  Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.
+  OPENAI_* and TANGLE_* equivalents are also accepted.
+
 Without arguments, prints this help.`;
 async function main() {
   const { command, positional, flags } = parseArgs(process.argv.slice(2));
+  assertKnownFlags(command, flags);
+  if (flags.help === "true") {
+    process.stdout.write(`${HELP}
+`);
+    return 0;
+  }
   switch (command) {
     case "serve": {
-      const port = Number(flags.port ?? 5005);
+      const port = parsePort(flags.port ?? "5005");
       const host = flags.host ?? "127.0.0.1";
-      const server = startServer({ port, host });
+      const llm = resolveCliLlmConfig();
+      const { server } = await startServerAsync({
+        port,
+        host,
+        llm: llm.client,
+        judgeModel: llm.model,
+        llmRouteRequirements: { requireExplicitBaseUrl: true }
+      });
       const shutdown = (sig) => {
         console.log(`[agent-eval] received ${sig}, shutting down`);
         server.close(() => process.exit(0));
@@ -72,11 +120,21 @@ async function main() {
     }
     case "rpc": {
       const [method] = positional;
-      return await runRpcOnce(method);
+      const llm = resolveCliLlmConfig();
+      return await runRpcOnce(method, {
+        llm: llm.client,
+        judgeModel: llm.model,
+        llmRouteRequirements: { requireExplicitBaseUrl: true }
+      });
     }
     case "rpc-batch": {
       const [method] = positional;
-      return await runRpcBatch(method);
+      const llm = resolveCliLlmConfig();
+      return await runRpcBatch(method, {
+        llm: llm.client,
+        judgeModel: llm.model,
+        llmRouteRequirements: { requireExplicitBaseUrl: true }
+      });
     }
     case "openapi": {
       const out = flags.out ?? "openapi.json";
@@ -88,6 +146,11 @@ async function main() {
     }
     case "version": {
       process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}
+`);
+      return 0;
+    }
+    case "--version": {
+      process.stdout.write(`${handleVersion().version}
 `);
       return 0;
     }
@@ -105,6 +168,33 @@ ${HELP}
       return 1;
   }
 }
+var FLAGS_BY_COMMAND = {
+  serve: /* @__PURE__ */ new Set(["help", "host", "port"]),
+  rpc: /* @__PURE__ */ new Set(["help"]),
+  "rpc-batch": /* @__PURE__ */ new Set(["help"]),
+  openapi: /* @__PURE__ */ new Set(["help", "out"]),
+  version: /* @__PURE__ */ new Set(["help"]),
+  help: /* @__PURE__ */ new Set(),
+  "--help": /* @__PURE__ */ new Set(),
+  "-h": /* @__PURE__ */ new Set(),
+  "--version": /* @__PURE__ */ new Set(),
+  "": /* @__PURE__ */ new Set()
+};
+function assertKnownFlags(command, flags) {
+  const allowed = FLAGS_BY_COMMAND[command];
+  if (!allowed) return;
+  const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag));
+  if (unknown.length > 0) {
+    throw new Error(`unknown flag for ${command || "help"}: --${unknown[0]}`);
+  }
+}
+function parsePort(raw) {
+  const port = Number(raw);
+  if (!Number.isInteger(port) || port < 0 || port > 65535) {
+    throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`);
+  }
+  return port;
+}
 main().then((code) => process.exit(code)).catch((err) => {
   console.error("[agent-eval] cli error:", err);
   process.exit(1);

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServer } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]!\n    if (tok.startsWith('--')) {\n      const key = tok.slice(2)\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval — wire-protocol entry point.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Method ∈ {judge, listRubrics, version}.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  version\n        Print server + wire-protocol version JSON.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n\n  switch (command) {\n    case 'serve': {\n      const port = Number(flags.port ?? 5005)\n      const host = flags.host ?? '127.0.0.1'\n      const server = startServer({ port, host })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      return await runRpcOnce(method)\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      return await runRpcBatch(method)\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'version': {\n      process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(`${HELP}\\n`)\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n"],"mappings":";;;;;;;;;;;;;;;;AAaA,SAAS,qBAAqB;AAY9B,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAEtE,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,OAAO,MAAM,QAAQ,IAAI;AACtC,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,SAAS,YAAY,EAAE,MAAM,KAAK,CAAC;AAEzC,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,WAAW,MAAM;AAAA,IAChC;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,YAAY,MAAM;AAAA,IACjC;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../src/cli.ts","../src/cli-config.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]!\n    if (tok.startsWith('--')) {\n      const raw = tok.slice(2)\n      const equalsAt = raw.indexOf('=')\n      if (equalsAt >= 0) {\n        flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n        continue\n      }\n      const key = raw\n      if (key === 'help') {\n        flags[key] = 'true'\n        continue\n      }\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else if (tok === '-h') {\n      flags.help = 'true'\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Methods: judge, listRubrics, version.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  version\n        Print server + wire-protocol version JSON.\n\nJudge provider:\n  Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n  OPENAI_* and TANGLE_* equivalents are also accepted.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n  assertKnownFlags(command, flags)\n\n  if (flags.help === 'true') {\n    process.stdout.write(`${HELP}\\n`)\n    return 0\n  }\n\n  switch (command) {\n    case 'serve': {\n      const port = parsePort(flags.port ?? '5005')\n      const host = flags.host ?? '127.0.0.1'\n      const llm = resolveCliLlmConfig()\n      const { server } = await startServerAsync({\n        port,\n        host,\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcOnce(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcBatch(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'version': {\n      process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n      return 0\n    }\n    case '--version': {\n      process.stdout.write(`${handleVersion().version}\\n`)\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(`${HELP}\\n`)\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n  serve: new Set(['help', 'host', 'port']),\n  rpc: new Set(['help']),\n  'rpc-batch': new Set(['help']),\n  openapi: new Set(['help', 'out']),\n  version: new Set(['help']),\n  help: new Set(),\n  '--help': new Set(),\n  '-h': new Set(),\n  '--version': new Set(),\n  '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n  const allowed = FLAGS_BY_COMMAND[command]\n  if (!allowed) return\n  const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n  if (unknown.length > 0) {\n    throw new Error(`unknown flag for ${command || 'help'}: --${unknown[0]}`)\n  }\n}\n\nfunction parsePort(raw: string): number {\n  const port = Number(raw)\n  if (!Number.isInteger(port) || port < 0 || port > 65_535) {\n    throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n  }\n  return port\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n","import type { LlmClientOptions } from './llm-client'\n\nexport interface CliLlmConfig {\n  client?: LlmClientOptions\n  model?: string\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n  const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n  const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n  const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n  const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n  const baseUrl =\n    explicitBaseUrl ??\n    nonEmpty(env.OPENAI_BASE_URL) ??\n    nonEmpty(env.TANGLE_ROUTER_URL) ??\n    (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n    (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n  const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n  const model =\n    nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n  const client =\n    baseUrl || apiKey\n      ? { ...(baseUrl ? { baseUrl } : {}), ...(apiKey ? { apiKey } : {}) }\n      : undefined\n  return { ...(client ? { client } : {}), ...(model ? { model } : {}) }\n}\n\nfunction nonEmpty(value: string | undefined): string | undefined {\n  const trimmed = value?.trim()\n  return trimmed ? trimmed : undefined\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAaA,SAAS,qBAAqB;;;ACNvB,SAAS,oBAAoB,MAAyB,QAAQ,KAAmB;AACtF,QAAM,kBAAkB,SAAS,IAAI,uBAAuB;AAC5D,QAAM,iBAAiB,SAAS,IAAI,sBAAsB;AAC1D,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,YAC7C,eAAe,mCAAmC;AACrD,QAAM,SAAS,kBAAkB,gBAAgB;AACjD,QAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;AAE/F,QAAM,SACJ,WAAW,SACP,EAAE,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC,GAAI,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,EAAG,IACjE;AACN,SAAO,EAAE,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,GAAI,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAG;AACtE;AAEA,SAAS,SAAS,OAA+C;AAC/D,QAAM,UAAU,OAAO,KAAK;AAC5B,SAAO,UAAU,UAAU;AAC7B;;;ADNA,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,WAAW,IAAI,QAAQ,GAAG;AAChC,UAAI,YAAY,GAAG;AACjB,cAAM,IAAI,MAAM,GAAG,QAAQ,CAAC,IAAI,IAAI,MAAM,WAAW,CAAC;AACtD;AAAA,MACF;AACA,YAAM,MAAM;AACZ,UAAI,QAAQ,QAAQ;AAClB,cAAM,GAAG,IAAI;AACb;AAAA,MACF;AACA,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,WAAW,QAAQ,MAAM;AACvB,YAAM,OAAO;AAAA,IACf,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AACtE,mBAAiB,SAAS,KAAK;AAE/B,MAAI,MAAM,SAAS,QAAQ;AACzB,YAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,WAAO;AAAA,EACT;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;AAC3C,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,MAAM,oBAAoB;AAChC,YAAM,EAAE,OAAO,IAAI,MAAM,iBAAiB;AAAA,QACxC;AAAA,QACA;AAAA,QACA,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAED,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,WAAW,QAAQ;AAAA,QAC9B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,YAAY,QAAQ;AAAA,QAC/B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK,aAAa;AAChB,cAAQ,OAAO,MAAM,GAAG,cAAc,EAAE,OAAO;AAAA,CAAI;AACnD,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,IAAM,mBAAwD;AAAA,EAC5D,OAAO,oBAAI,IAAI,CAAC,QAAQ,QAAQ,MAAM,CAAC;AAAA,EACvC,KAAK,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACrB,aAAa,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EAC7B,SAAS,oBAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;AAAA,EAChC,SAAS,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACzB,MAAM,oBAAI,IAAI;AAAA,EACd,UAAU,oBAAI,IAAI;AAAA,EAClB,MAAM,oBAAI,IAAI;AAAA,EACd,aAAa,oBAAI,IAAI;AAAA,EACrB,IAAI,oBAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;AAC9E,QAAM,UAAU,iBAAiB,OAAO;AACxC,MAAI,CAAC,QAAS;AACd,QAAM,UAAU,OAAO,KAAK,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;AACtE,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI,MAAM,oBAAoB,WAAW,MAAM,OAAO,QAAQ,CAAC,CAAC,EAAE;AAAA,EAC1E;AACF;AAEA,SAAS,UAAU,KAAqB;AACtC,QAAM,OAAO,OAAO,GAAG;AACvB,MAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAAQ;AACxD,UAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EAC9F;AACA,SAAO;AACT;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/contract/index.d.ts

@@ -1,11 +1,12 @@
+import { z } from 'zod';
 import { AgentCandidateExperiment, AgentCandidateBundle, AgentCandidateBenchmarkTask, AgentCandidateBenchmarkCellRef, AgentCandidateExperimentMeasurement, AgentImprovementMeasuredComparison, CandidateExecutionEvidence, AgentCandidateBenchmarkSuiteInputs, AgentCandidateBenchmarkTaskMaterial, AgentCandidateExperimentMaterial } from '@tangle-network/agent-interface';
 import { AxFunction, AxAIService } from '@ax-llm/ax';
-import { z } from 'zod';
 
 type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
-type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | {
+type AgentProfileJsonObject = {
     [key: string]: AgentProfileJson;
 };
+type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | AgentProfileJsonObject;
 type AgentProfileDimensionValue = string | number | boolean | null;
 interface AgentProfileSource {
     /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
@@ -75,9 +76,16 @@ interface CostReceiptInput extends CostUsage {
     costUnknown?: boolean;
     usageUnknown?: boolean;
 }
+/** Per-million token rates for a model or endpoint not covered by package pricing. */
+interface CustomTokenPricing {
+    inputUsdPerMillion: number;
+    outputUsdPerMillion: number;
+}
 type MaximumCharge = {
     externallyEnforcedMaximumUsd: number;
 } | ({
+    customTokenPricing: CustomTokenPricing;
+} & Pick<CostUsage, 'inputTokens' | 'outputTokens'>) | ({
     model: string;
 } & CostUsage);
 interface RunPaidCallInput<T> {
@@ -89,7 +97,7 @@ interface RunPaidCallInput<T> {
     model?: string;
     tags?: Record<string, string>;
     signal?: AbortSignal;
-    /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
+    /** Provider-enforced dollar maximum, or maximum token usage with known pricing. Required when capped. */
     maximumCharge?: MaximumCharge;
     /** `callId` can be forwarded as the provider's idempotency key. */
     execute(signal: AbortSignal, callId: string): Promise<T>;
@@ -859,8 +867,8 @@ interface LlmCallResult {
     content: string;
     usage: LlmUsage;
     /**
-     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
-     * `null` when neither the proxy nor the caller can derive it.
+     * Cost in USD. Uses the provider's reported cost when present, otherwise
+     * caller-supplied token pricing. `null` when neither is available.
      */
     costUsd: number | null;
     /** Model name actually used (echoed from response). */
@@ -919,6 +927,21 @@ interface LlmClientOptions {
     deadlineMs?: number;
     /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
     maxRetries?: number;
+    /** Token rates used when the provider omits cost or package pricing does not cover the model. */
+    customTokenPricing?: CustomTokenPricing;
+    /**
+     * Transport for requests that declare `jsonSchema`. `native` sends
+     * `response_format: json_schema`; `json-object` sends the broadly supported
+     * JSON mode and relies on the caller to include the schema in model-visible
+     * instructions. Default: `native`.
+     */
+    jsonSchemaTransport?: 'native' | 'json-object';
+    /**
+     * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.
+     * `exact` requires the complete response content to be one JSON value.
+     * Default: `extract`.
+     */
+    jsonPayloadMode?: 'extract' | 'exact';
     /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
     fetch?: typeof fetch;
     /**
@@ -2438,6 +2461,84 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
  */
 declare function runImprovementLoop<TScenario extends Scenario, TArtifact>(opts: RunImprovementLoopOptions<TScenario, TArtifact>): Promise<RunImprovementLoopResult<TArtifact, TScenario>>;
 
+/**
+ * `llmJudge` — the single-LLM-call bridge that turns a rubric prompt into a
+ * canonical campaign `JudgeConfig`.
+ *
+ * The `JudgeConfig` contract (src/campaign/types.ts) is deliberately a
+ * function, not a fixed LLM-prompt shape: real consumers judge with
+ * ensembles, deterministic checks, or one LLM call. `ensembleJudge`
+ * (src/judge-panel.ts) covers the multi-model case; `buildAgreementJudge`
+ * (src/campaign/distillation) covers the pure-comparator case. `llmJudge`
+ * covers the common single-call case the `JudgeConfig` doc-comment names:
+ * one model call against `prompt`, parsed into the canonical `JudgeScore`
+ * (`{ dimensions, composite, notes }`) on the campaign [0,1] scale.
+ *
+ * Transport is injected as a `ChatClient` (src/analyst/chat-client.ts) — the
+ * substrate's transport-agnostic LLM seam — so the judge stays decoupled from
+ * router-vs-sandbox-vs-cli-bridge and is unit-testable with the `mock`
+ * transport. The composite is computed by `weightedComposite` (the same
+ * sum-normalized weighting `ensembleJudge` uses), so a lift is attributable to
+ * the dimension scores, not to a bespoke reducer.
+ *
+ * Fail-loud throughout: an unparseable model response throws `JudgeParseError`;
+ * a response missing a declared dimension throws; an out-of-range score throws.
+ * A thrown judge is recorded by the campaign engine as a failed cell, never
+ * folded into a silent zero.
+ */
+
+/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
+ *  bare string uses the key as its own description. */
+type LlmJudgeDimension = string | JudgeDimension;
+interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {
+    /** The injected LLM transport. One `chat()` call per `score()`. Required —
+     *  there is no default route, so a misconfigured judge fails at construction,
+     *  never silently against the free-tier router. */
+    chat: ChatClient;
+    /** Rubric dimensions the model scores. Each becomes a `[0,1]` field of the
+     *  returned `JudgeScore.dimensions`. Defaults to a single `quality` dimension. */
+    dimensions?: LlmJudgeDimension[];
+    /** Model id. Falls back to `chat.defaultModel`; one of the two MUST resolve. */
+    model?: string;
+    /** Explicit scoring revision for opaque transport or renderer changes. */
+    judgeVersion?: string;
+    temperature?: number;
+    maxTokens?: number;
+    /** Composite weights forwarded to `weightedComposite`: a partial map selects
+     *  AND weights exactly the named dimensions. Omit for a uniform mean. */
+    weights?: Record<string, number>;
+    /** Scale the model is prompted to score on, normalized into `[0,1]`:
+     *   - `'unit'`  (default): the model returns `[0,1]` directly.
+     *   - `'ten'`:  the model returns `[0,10]`; divided by 10 here.
+     *  The prompt is annotated with the expected range either way. */
+    scale?: 'unit' | 'ten';
+    /** Run this judge only on matching scenarios (mirrors `JudgeConfig.appliesTo`). */
+    appliesTo?: (scenario: TScenario) => boolean;
+    /** Render the artifact + scenario into the user message. Default:
+     *  pretty-printed JSON of `{ scenario, artifact }`. */
+    renderUser?: (input: {
+        artifact: TArtifact;
+        scenario: TScenario;
+    }) => string;
+    /** Strict runtime contract; its JSON Schema is sent to the provider. */
+    costLedger?: CostLedgerHandle;
+    responseSchema?: {
+        name: string;
+        schema: z.ZodObject;
+    };
+}
+/**
+ * Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call
+ * against `prompt` and reduces the model's per-dimension scores to a canonical
+ * `JudgeScore` in `[0,1]`.
+ *
+ * The model is instructed to return JSON `{ "dimensions": { <key>: <number>, … },
+ * "notes": "…" }`; the helper strips fenced JSON, validates every declared
+ * dimension is present and in range, normalizes by `scale`, and composites via
+ * `weightedComposite`.
+ */
+declare function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(name: string, prompt: string, opts: LlmJudgeOptions<TArtifact, TScenario>): JudgeConfig<TArtifact, TScenario>;
+
 declare const REFERENCE_EQUIVALENCE_JUDGE_VERSION = "reference-equivalence-judge-v1-2026-07-13";
 declare const REFERENCE_EQUIVALENCE_INPUT_LIMITS: {
     readonly userRequest: 8000;
@@ -3268,9 +3369,15 @@ interface InterRaterInsight {
     raters: number;
     /** Number of runs every rater scored. */
     jointlyRated: number;
-    /** Cohen's κ averaged across rater pairs. */
+    /** Multi-rater weighted kappa over the jointly rated runs. */
     kappa: number;
-    /** Pairwise κ per rater pair (key = `"raterA::raterB"`). */
+    /** Absolute agreement across raters, using ICC(2,1). */
+    icc: number;
+    /** Mean pairwise Pearson correlation. Correlation is not agreement. */
+    pearson: number;
+    /** Mean pairwise Spearman rank correlation. */
+    spearman: number;
+    /** Pairwise weighted kappa per rater pair (key = `"raterA::raterB"`). */
     perPair: Record<string, number>;
     /** Run ids where raters disagree the most — the high-value triage list. */
     disagreementCases: Array<{
@@ -5165,4 +5272,4 @@ interface FromOtelSpansOptions {
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type CandidateExperimentExecutionInput, type ChatClient, type CodeAgentSessionAction, type CodeAgentSessionActionKind, type CodeAgentSessionActionStatus, type CodeAgentSessionActionSurface, type CodeAgentSessionDiagnostic, type CodeAgentSessionExecutionReceipt, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionObservation, type CodeAgentSessionSource, type CodeAgentSessionTerminalStatus, type CodeSurface, type CompareCandidateExperimentOptions, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunCandidateExperimentOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SealCandidateBenchmarkSuiteOptions, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, summarizeExecution, verifyCandidateBenchmarkSuite, verifyCandidateBenchmarkSuiteInputs, verifyCandidateBenchmarkTask, verifyCandidateExperiment, verifyCandidateExperimentComparison };
+export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type CandidateExperimentExecutionInput, type ChatClient, type CodeAgentSessionAction, type CodeAgentSessionActionKind, type CodeAgentSessionActionStatus, type CodeAgentSessionActionSurface, type CodeAgentSessionDiagnostic, type CodeAgentSessionExecutionReceipt, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionObservation, type CodeAgentSessionSource, type CodeAgentSessionTerminalStatus, type CodeSurface, type CompareCandidateExperimentOptions, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type LlmJudgeDimension, type LlmJudgeOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunCandidateExperimentOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SealCandidateBenchmarkSuiteOptions, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, llmJudge, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, summarizeExecution, verifyCandidateBenchmarkSuite, verifyCandidateBenchmarkSuiteInputs, verifyCandidateBenchmarkTask, verifyCandidateExperiment, verifyCandidateExperimentComparison };

package/dist/contract/index.js

@@ -14,7 +14,7 @@ import {
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-6WX7CBAR.js";
+} from "../chunk-SJT4OBVL.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -27,6 +27,7 @@ import {
   gepaProposer,
   heldOutGate,
   heldoutSignificance,
+  llmJudge,
   loopProvenanceArgsFromResult,
   paretoPolicy,
   paretoSignificanceGate,
@@ -36,7 +37,7 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-HZJF4IUO.js";
+} from "../chunk-SUN7QLPB.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -44,24 +45,24 @@ import {
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-3FCG7FBV.js";
+} from "../chunk-D5JZ7UDZ.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-VJ7T5WIO.js";
+} from "../chunk-WP3GEXKI.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-FC5NDO3E.js";
+import "../chunk-MHPEGJHC.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "../chunk-3RF76KTD.js";
 import "../chunk-ARU2PZFM.js";
-import "../chunk-NJC7U437.js";
+import "../chunk-MK3Q6GCL.js";
 import "../chunk-DPZAEKA6.js";
 import {
   pairedBootstrap
 } from "../chunk-PJQFMIOX.js";
-import "../chunk-BGVTIE2C.js";
+import "../chunk-GRCDRKII.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
@@ -74,9 +75,9 @@ import "../chunk-IR3KBHOY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
-} from "../chunk-S3UZOQ5Y.js";
+} from "../chunk-LOW3U7JZ.js";
 import "../chunk-MA6HLL3S.js";
-import "../chunk-XJYR7XFV.js";
+import "../chunk-GC4ATIKK.js";
 import "../chunk-VSMTAMNK.js";
 import "../chunk-ONWEPEDO.js";
 import {
@@ -1703,6 +1704,7 @@ export {
   gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
+  llmJudge,
   measuredComparisonFromCandidateExperiment,
   observeCodeAgentSession,
   paretoPolicy,