@tangle-network/agent-eval 0.123.8 → 0.125.0

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts","../src/cli-config.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]!\n    if (tok.startsWith('--')) {\n      const raw = tok.slice(2)\n      const equalsAt = raw.indexOf('=')\n      if (equalsAt >= 0) {\n        flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n        continue\n      }\n      const key = raw\n      if (key === 'help') {\n        flags[key] = 'true'\n        continue\n      }\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else if (tok === '-h') {\n      flags.help = 'true'\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Methods: judge, listRubrics, version.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  version\n        Print server + wire-protocol version JSON.\n\nJudge provider:\n  Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n  OPENAI_* and TANGLE_* equivalents are also accepted.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n  assertKnownFlags(command, flags)\n\n  if (flags.help === 'true') {\n    process.stdout.write(`${HELP}\\n`)\n    return 0\n  }\n\n  switch (command) {\n    case 'serve': {\n      const port = parsePort(flags.port ?? '5005')\n      const host = flags.host ?? '127.0.0.1'\n      const llm = resolveCliLlmConfig()\n      const { server } = await startServerAsync({\n        port,\n        host,\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcOnce(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcBatch(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'version': {\n      process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n      return 0\n    }\n    case '--version': {\n      process.stdout.write(`${handleVersion().version}\\n`)\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(`${HELP}\\n`)\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n  serve: new Set(['help', 'host', 'port']),\n  rpc: new Set(['help']),\n  'rpc-batch': new Set(['help']),\n  openapi: new Set(['help', 'out']),\n  version: new Set(['help']),\n  help: new Set(),\n  '--help': new Set(),\n  '-h': new Set(),\n  '--version': new Set(),\n  '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n  const allowed = FLAGS_BY_COMMAND[command]\n  if (!allowed) return\n  const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n  if (unknown.length > 0) {\n    throw new Error(`unknown flag for ${command || 'help'}: --${unknown[0]}`)\n  }\n}\n\nfunction parsePort(raw: string): number {\n  const port = Number(raw)\n  if (!Number.isInteger(port) || port < 0 || port > 65_535) {\n    throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n  }\n  return port\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n","import type { LlmClientOptions } from './llm-client'\n\nexport interface CliLlmConfig {\n  client?: LlmClientOptions\n  model?: string\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n  const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n  const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n  const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n  const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n  const baseUrl =\n    explicitBaseUrl ??\n    nonEmpty(env.OPENAI_BASE_URL) ??\n    nonEmpty(env.TANGLE_ROUTER_URL) ??\n    (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n    (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n  const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n  const model =\n    nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n  const client =\n    baseUrl || apiKey\n      ? { ...(baseUrl ? { baseUrl } : {}), ...(apiKey ? { apiKey } : {}) }\n      : undefined\n  return { ...(client ? { client } : {}), ...(model ? { model } : {}) }\n}\n\nfunction nonEmpty(value: string | undefined): string | undefined {\n  const trimmed = value?.trim()\n  return trimmed ? trimmed : undefined\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAaA,SAAS,qBAAqB;;;ACNvB,SAAS,oBAAoB,MAAyB,QAAQ,KAAmB;AACtF,QAAM,kBAAkB,SAAS,IAAI,uBAAuB;AAC5D,QAAM,iBAAiB,SAAS,IAAI,sBAAsB;AAC1D,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,YAC7C,eAAe,mCAAmC;AACrD,QAAM,SAAS,kBAAkB,gBAAgB;AACjD,QAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;AAE/F,QAAM,SACJ,WAAW,SACP,EAAE,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC,GAAI,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,EAAG,IACjE;AACN,SAAO,EAAE,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,GAAI,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAG;AACtE;AAEA,SAAS,SAAS,OAA+C;AAC/D,QAAM,UAAU,OAAO,KAAK;AAC5B,SAAO,UAAU,UAAU;AAC7B;;;ADNA,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,WAAW,IAAI,QAAQ,GAAG;AAChC,UAAI,YAAY,GAAG;AACjB,cAAM,IAAI,MAAM,GAAG,QAAQ,CAAC,IAAI,IAAI,MAAM,WAAW,CAAC;AACtD;AAAA,MACF;AACA,YAAM,MAAM;AACZ,UAAI,QAAQ,QAAQ;AAClB,cAAM,GAAG,IAAI;AACb;AAAA,MACF;AACA,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,WAAW,QAAQ,MAAM;AACvB,YAAM,OAAO;AAAA,IACf,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AACtE,mBAAiB,SAAS,KAAK;AAE/B,MAAI,MAAM,SAAS,QAAQ;AACzB,YAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,WAAO;AAAA,EACT;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;AAC3C,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,MAAM,oBAAoB;AAChC,YAAM,EAAE,OAAO,IAAI,MAAM,iBAAiB;AAAA,QACxC;AAAA,QACA;AAAA,QACA,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAED,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,WAAW,QAAQ;AAAA,QAC9B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,YAAY,QAAQ;AAAA,QAC/B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK,aAAa;AAChB,cAAQ,OAAO,MAAM,GAAG,cAAc,EAAE,OAAO;AAAA,CAAI;AACnD,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,IAAM,mBAAwD;AAAA,EAC5D,OAAO,oBAAI,IAAI,CAAC,QAAQ,QAAQ,MAAM,CAAC;AAAA,EACvC,KAAK,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACrB,aAAa,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EAC7B,SAAS,oBAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;AAAA,EAChC,SAAS,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACzB,MAAM,oBAAI,IAAI;AAAA,EACd,UAAU,oBAAI,IAAI;AAAA,EAClB,MAAM,oBAAI,IAAI;AAAA,EACd,aAAa,oBAAI,IAAI;AAAA,EACrB,IAAI,oBAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;AAC9E,QAAM,UAAU,iBAAiB,OAAO;AACxC,MAAI,CAAC,QAAS;AACd,QAAM,UAAU,OAAO,KAAK,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;AACtE,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI,MAAM,oBAAoB,WAAW,MAAM,OAAO,QAAQ,CAAC,CAAC,EAAE;AAAA,EAC1E;AACF;AAEA,SAAS,UAAU,KAAqB;AACtC,QAAM,OAAO,OAAO,GAAG;AACvB,MAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAAQ;AACxD,UAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EAC9F;AACA,SAAO;AACT;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../src/cli.ts","../src/cli-config.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { resolveCliLlmConfig } from './cli-config'\nimport { runRolloutReleaseCli } from './rollout/release/hf-dataset'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServerAsync } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]!\n    if (tok.startsWith('--')) {\n      const raw = tok.slice(2)\n      const equalsAt = raw.indexOf('=')\n      if (equalsAt >= 0) {\n        flags[raw.slice(0, equalsAt)] = raw.slice(equalsAt + 1)\n        continue\n      }\n      const key = raw\n      if (key === 'help') {\n        flags[key] = 'true'\n        continue\n      }\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else if (tok === '-h') {\n      flags.help = 'true'\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval: evaluation RPC and HTTP server.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Methods: judge, listRubrics, version.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]\n        Build a HuggingFace-ready dataset dir from tangle.rollout.v1 ledgers:\n        validate, fail-closed split filter, scrub, export formats + card.\n  version\n        Print server + wire-protocol version JSON.\n\nJudge provider:\n  Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL.\n  OPENAI_* and TANGLE_* equivalents are also accepted.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n  assertKnownFlags(command, flags)\n\n  if (flags.help === 'true') {\n    process.stdout.write(`${HELP}\\n`)\n    return 0\n  }\n\n  switch (command) {\n    case 'serve': {\n      const port = parsePort(flags.port ?? '5005')\n      const host = flags.host ?? '127.0.0.1'\n      const llm = resolveCliLlmConfig()\n      const { server } = await startServerAsync({\n        port,\n        host,\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcOnce(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      const llm = resolveCliLlmConfig()\n      return await runRpcBatch(method, {\n        llm: llm.client,\n        judgeModel: llm.model,\n        llmRouteRequirements: { requireExplicitBaseUrl: true },\n      })\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'rollout-release': {\n      // The subcommand owns its own flag grammar (multi-value --formats,\n      // boolean --include-proposers) — pass raw argv through untouched.\n      return await runRolloutReleaseCli(process.argv.slice(3))\n    }\n    case 'version': {\n      process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n      return 0\n    }\n    case '--version': {\n      process.stdout.write(`${handleVersion().version}\\n`)\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(`${HELP}\\n`)\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nconst FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {\n  serve: new Set(['help', 'host', 'port']),\n  rpc: new Set(['help']),\n  'rpc-batch': new Set(['help']),\n  openapi: new Set(['help', 'out']),\n  version: new Set(['help']),\n  help: new Set(),\n  '--help': new Set(),\n  '-h': new Set(),\n  '--version': new Set(),\n  '': new Set(),\n}\n\nfunction assertKnownFlags(command: string, flags: Record<string, string>): void {\n  // rollout-release parses its own argv (multi-value flags).\n  if (command === 'rollout-release') return\n  const allowed = FLAGS_BY_COMMAND[command]\n  if (!allowed) return\n  const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))\n  if (unknown.length > 0) {\n    throw new Error(`unknown flag for ${command || 'help'}: --${unknown[0]}`)\n  }\n}\n\nfunction parsePort(raw: string): number {\n  const port = Number(raw)\n  if (!Number.isInteger(port) || port < 0 || port > 65_535) {\n    throw new Error(`--port must be an integer from 0 to 65535; received ${JSON.stringify(raw)}`)\n  }\n  return port\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n","import type { LlmClientOptions } from './llm-client'\n\nexport interface CliLlmConfig {\n  client?: LlmClientOptions\n  model?: string\n}\n\nexport function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig {\n  const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL)\n  const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY)\n  const openAiApiKey = nonEmpty(env.OPENAI_API_KEY)\n  const tangleApiKey = nonEmpty(env.TANGLE_API_KEY)\n  const baseUrl =\n    explicitBaseUrl ??\n    nonEmpty(env.OPENAI_BASE_URL) ??\n    nonEmpty(env.TANGLE_ROUTER_URL) ??\n    (openAiApiKey ? 'https://api.openai.com/v1' : undefined) ??\n    (tangleApiKey ? 'https://router.tangle.tools/v1' : undefined)\n  const apiKey = explicitApiKey ?? openAiApiKey ?? tangleApiKey\n  const model =\n    nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL)\n\n  const client =\n    baseUrl || apiKey\n      ? { ...(baseUrl ? { baseUrl } : {}), ...(apiKey ? { apiKey } : {}) }\n      : undefined\n  return { ...(client ? { client } : {}), ...(model ? { model } : {}) }\n}\n\nfunction nonEmpty(value: string | undefined): string | undefined {\n  const trimmed = value?.trim()\n  return trimmed ? trimmed : undefined\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAaA,SAAS,qBAAqB;;;ACNvB,SAAS,oBAAoB,MAAyB,QAAQ,KAAmB;AACtF,QAAM,kBAAkB,SAAS,IAAI,uBAAuB;AAC5D,QAAM,iBAAiB,SAAS,IAAI,sBAAsB;AAC1D,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,eAAe,SAAS,IAAI,cAAc;AAChD,QAAM,UACJ,mBACA,SAAS,IAAI,eAAe,KAC5B,SAAS,IAAI,iBAAiB,MAC7B,eAAe,8BAA8B,YAC7C,eAAe,mCAAmC;AACrD,QAAM,SAAS,kBAAkB,gBAAgB;AACjD,QAAM,QACJ,SAAS,IAAI,oBAAoB,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY;AAE/F,QAAM,SACJ,WAAW,SACP,EAAE,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC,GAAI,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,EAAG,IACjE;AACN,SAAO,EAAE,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,GAAI,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC,EAAG;AACtE;AAEA,SAAS,SAAS,OAA+C;AAC/D,QAAM,UAAU,OAAO,KAAK;AAC5B,SAAO,UAAU,UAAU;AAC7B;;;ADLA,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,WAAW,IAAI,QAAQ,GAAG;AAChC,UAAI,YAAY,GAAG;AACjB,cAAM,IAAI,MAAM,GAAG,QAAQ,CAAC,IAAI,IAAI,MAAM,WAAW,CAAC;AACtD;AAAA,MACF;AACA,YAAM,MAAM;AACZ,UAAI,QAAQ,QAAQ;AAClB,cAAM,GAAG,IAAI;AACb;AAAA,MACF;AACA,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,WAAW,QAAQ,MAAM;AACvB,YAAM,OAAO;AAAA,IACf,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AACtE,mBAAiB,SAAS,KAAK;AAE/B,MAAI,MAAM,SAAS,QAAQ;AACzB,YAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,WAAO;AAAA,EACT;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,UAAU,MAAM,QAAQ,MAAM;AAC3C,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,MAAM,oBAAoB;AAChC,YAAM,EAAE,OAAO,IAAI,MAAM,iBAAiB;AAAA,QACxC;AAAA,QACA;AAAA,QACA,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAED,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,WAAW,QAAQ;AAAA,QAC9B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,YAAM,MAAM,oBAAoB;AAChC,aAAO,MAAM,YAAY,QAAQ;AAAA,QAC/B,KAAK,IAAI;AAAA,QACT,YAAY,IAAI;AAAA,QAChB,sBAAsB,EAAE,wBAAwB,KAAK;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,mBAAmB;AAGtB,aAAO,MAAM,qBAAqB,QAAQ,KAAK,MAAM,CAAC,CAAC;AAAA,IACzD;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK,aAAa;AAChB,cAAQ,OAAO,MAAM,GAAG,cAAc,EAAE,OAAO;AAAA,CAAI;AACnD,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,IAAM,mBAAwD;AAAA,EAC5D,OAAO,oBAAI,IAAI,CAAC,QAAQ,QAAQ,MAAM,CAAC;AAAA,EACvC,KAAK,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACrB,aAAa,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EAC7B,SAAS,oBAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;AAAA,EAChC,SAAS,oBAAI,IAAI,CAAC,MAAM,CAAC;AAAA,EACzB,MAAM,oBAAI,IAAI;AAAA,EACd,UAAU,oBAAI,IAAI;AAAA,EAClB,MAAM,oBAAI,IAAI;AAAA,EACd,aAAa,oBAAI,IAAI;AAAA,EACrB,IAAI,oBAAI,IAAI;AACd;AAEA,SAAS,iBAAiB,SAAiB,OAAqC;AAE9E,MAAI,YAAY,kBAAmB;AACnC,QAAM,UAAU,iBAAiB,OAAO;AACxC,MAAI,CAAC,QAAS;AACd,QAAM,UAAU,OAAO,KAAK,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;AACtE,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI,MAAM,oBAAoB,WAAW,MAAM,OAAO,QAAQ,CAAC,CAAC,EAAE;AAAA,EAC1E;AACF;AAEA,SAAS,UAAU,KAAqB;AACtC,QAAM,OAAO,OAAO,GAAG;AACvB,MAAI,CAAC,OAAO,UAAU,IAAI,KAAK,OAAO,KAAK,OAAO,OAAQ;AACxD,UAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EAC9F;AACA,SAAO;AACT;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/contract/index.d.ts

@@ -838,6 +838,7 @@ interface LlmMessage {
         };
     }>;
 }
+type LlmThinkingMode = 'enabled' | 'disabled';
 interface LlmCallRequest {
     model: string;
     messages: LlmMessage[];
@@ -850,6 +851,8 @@ interface LlmCallRequest {
     };
     temperature?: number;
     maxTokens?: number;
+    /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */
+    thinking?: LlmThinkingMode;
     /** Per-call timeout, default 300s. */
     timeoutMs?: number;
 }
@@ -944,6 +947,8 @@ interface LlmClientOptions {
      * Default: `extract`.
      */
     jsonPayloadMode?: 'extract' | 'exact';
+    /** Default provider reasoning mode. A per-call request value takes precedence. */
+    thinking?: LlmThinkingMode;
     /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
     fetch?: typeof fetch;
     /**
@@ -2664,6 +2669,8 @@ interface GepaProposerOptions {
     temperature?: number;
     /** Reflection max tokens. Default 6000. */
     maxTokens?: number;
+    /** Provider reasoning mode for each reflection call. Omitted to preserve the provider default. */
+    thinking?: LlmCallRequest['thinking'];
     /** Structured-doc constraints. Candidates violating any are rejected
      *  post-parse and dropped from the returned population. */
     constraints?: GepaProposerConstraints;

package/dist/contract/index.js

@@ -37,7 +37,7 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-QVGVJQMR.js";
+} from "../chunk-PMITBABE.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -49,15 +49,15 @@ import {
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-BUR5R4R4.js";
+} from "../chunk-A6GT67HT.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-EEHHBAWH.js";
+import "../chunk-DT7OXY3C.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "../chunk-3RF76KTD.js";
 import "../chunk-ARU2PZFM.js";
-import "../chunk-PDHIOKRE.js";
+import "../chunk-EQUK3RFS.js";
 import "../chunk-DPZAEKA6.js";
 import {
   pairedBootstrap