@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/chunk-YUFXO3TU.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/run-record.ts"],"sourcesContent":["/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag and either a `searchScore` or a `holdoutScore`.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n *   const rec = validateRunRecord(rawJson)         // throws on missing\n *   const ok  = isRunRecord(rawJson)               // boolean check\n *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency. Round-trip tested in `tests/run-record.test.ts`.\n */\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n *  combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\nexport interface RunTokenUsage {\n  input: number\n  output: number\n  cached?: number\n}\n\nexport interface RunJudgeMetadata {\n  model: string\n  promptVersion: string\n  /** [0,1] confidence the judge declared. Constant judge confidence\n   *  across many runs is a fallback signal (see `canary.ts`). */\n  confidence: number\n  /** True if the judge degraded to a fallback path (rules-only,\n   *  prior-call cache, etc.). The canary uses this to alert. */\n  fallback: boolean\n}\n\nexport interface RunOutcome {\n  /** Score on the search/optimization split. Optional because a\n   *  holdout-only evaluation only fills `holdoutScore`. */\n  searchScore?: number\n  /** Score on the held-out split. Optional because a search-only run\n   *  only fills `searchScore`. At least one must be present. */\n  holdoutScore?: number\n  /** Bag of any other metric the run produced — judge dimensions,\n   *  pass/fail counters, latency stats, etc. Numeric only — keeps\n   *  reporters honest. */\n  raw: Record<string, number>\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n *     model (after any steering bundle merge).\n *   - `configHash` is the sha256 of the effective run config (model,\n *     temperature, tools, judges, splits). The pair (promptHash,\n *     configHash) uniquely identifies an experimental cell.\n *\n * Model snapshot discipline:\n *   - `model` MUST encode a snapshot version. Bare aliases like\n *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n */\nexport interface RunRecord {\n  /** UUID for the run. */\n  runId: string\n  /** Logical experiment grouping (a treatment vs a baseline within\n   *  the same sweep should share `experimentId`). */\n  experimentId: string\n  /** Stable identifier for the candidate (variant) being run. The\n   *  promotion gate compares two `candidateId`s on matched items. */\n  candidateId: string\n  /** RNG seed for the run. Always recorded — silent re-seeding is\n   *  the most common cause of non-reproducible numbers. */\n  seed: number\n  /** Model identifier WITH snapshot version. */\n  model: string\n  /** sha256 of the effective prompt (post-steering). */\n  promptHash: string\n  /** sha256 of the effective config. */\n  configHash: string\n  /** Git SHA the harness was run from. */\n  commitSha: string\n  /** End-to-end wall-clock duration in milliseconds. */\n  wallMs: number\n  /** Time spent queued before execution started, if known. */\n  queueMs?: number\n  /** Total USD cost. Mandatory — runs without a cost number are\n   *  unbounded by definition and must not be admitted into the gate. */\n  costUsd: number\n  /** Token usage breakdown. */\n  tokenUsage: RunTokenUsage\n  /** Judge-side metadata, if a judge was used. */\n  judgeMetadata?: RunJudgeMetadata\n  /** Per-split scores + raw bag. */\n  outcome: RunOutcome\n  /** Categorical failure tag, when the run failed and the harness\n   *  classified it. Free-form string; standard tags live in\n   *  `failure-taxonomy.ts`. */\n  failureMode?: string\n  /** Which split this run was drawn from. */\n  splitTag: RunSplitTag\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n  'runId',\n  'experimentId',\n  'candidateId',\n  'seed',\n  'model',\n  'promptHash',\n  'configHash',\n  'commitSha',\n  'wallMs',\n  'costUsd',\n  'tokenUsage',\n  'outcome',\n  'splitTag',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\n\nexport class RunRecordValidationError extends Error {\n  readonly path: string\n  constructor(message: string, path = '') {\n    super(path ? `${message} (at ${path})` : message)\n    this.name = 'RunRecordValidationError'\n    this.path = path\n  }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n  if (input === null || typeof input !== 'object') {\n    throw new RunRecordValidationError('expected object')\n  }\n  const obj = input as Record<string, unknown>\n\n  for (const key of MANDATORY_TOP_LEVEL) {\n    if (!(key in obj)) {\n      throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n    }\n  }\n\n  expectString(obj.runId, 'runId')\n  expectString(obj.experimentId, 'experimentId')\n  expectString(obj.candidateId, 'candidateId')\n  expectFiniteNumber(obj.seed, 'seed')\n  expectString(obj.model, 'model')\n  expectString(obj.promptHash, 'promptHash')\n  expectString(obj.configHash, 'configHash')\n  expectString(obj.commitSha, 'commitSha')\n  expectFiniteNumber(obj.wallMs, 'wallMs')\n  if (obj.queueMs !== undefined) expectFiniteNumber(obj.queueMs, 'queueMs')\n  expectFiniteNumber(obj.costUsd, 'costUsd')\n\n  // Snapshot discipline: bare model aliases are not paper-grade.\n  if (!modelHasSnapshot(obj.model as string)) {\n    throw new RunRecordValidationError(\n      `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD')`,\n      'model',\n    )\n  }\n\n  // Token usage.\n  const tu = obj.tokenUsage\n  if (tu === null || typeof tu !== 'object') {\n    throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n  }\n  const tuRec = tu as Record<string, unknown>\n  expectFiniteNumber(tuRec.input, 'tokenUsage.input')\n  expectFiniteNumber(tuRec.output, 'tokenUsage.output')\n  if (tuRec.cached !== undefined) expectFiniteNumber(tuRec.cached, 'tokenUsage.cached')\n\n  // Judge metadata, optional.\n  if (obj.judgeMetadata !== undefined) {\n    const jm = obj.judgeMetadata\n    if (jm === null || typeof jm !== 'object') {\n      throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n    }\n    const jmRec = jm as Record<string, unknown>\n    expectString(jmRec.model, 'judgeMetadata.model')\n    expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n    expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n    if (typeof jmRec.fallback !== 'boolean') {\n      throw new RunRecordValidationError('judgeMetadata.fallback must be boolean', 'judgeMetadata.fallback')\n    }\n  }\n\n  // Outcome.\n  const out = obj.outcome\n  if (out === null || typeof out !== 'object') {\n    throw new RunRecordValidationError('outcome must be an object', 'outcome')\n  }\n  const outRec = out as Record<string, unknown>\n  if (outRec.searchScore !== undefined) expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n  if (outRec.holdoutScore !== undefined) expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n  if (outRec.searchScore === undefined && outRec.holdoutScore === undefined) {\n    throw new RunRecordValidationError(\n      'outcome must define searchScore or holdoutScore (or both)',\n      'outcome',\n    )\n  }\n  const raw = outRec.raw\n  if (raw === null || typeof raw !== 'object') {\n    throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n  }\n  for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n    expectFiniteNumber(v, `outcome.raw.${k}`)\n  }\n\n  // Failure mode optional.\n  if (obj.failureMode !== undefined) expectString(obj.failureMode, 'failureMode')\n\n  // Split tag.\n  if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n    throw new RunRecordValidationError(\n      `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n      'splitTag',\n    )\n  }\n\n  return input as RunRecord\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n  try {\n    validateRunRecord(input)\n    return true\n  } catch {\n    return false\n  }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n  input: unknown,\n):\n  | { ok: true; value: RunRecord }\n  | { ok: false; error: RunRecordValidationError } {\n  try {\n    return { ok: true, value: validateRunRecord(input) }\n  } catch (e) {\n    if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n    throw e\n  }\n}\n\n/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */\nexport function roundTripRunRecord(record: RunRecord): RunRecord {\n  const json = JSON.stringify(record)\n  return validateRunRecord(JSON.parse(json))\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n  if (typeof value !== 'string' || value.length === 0) {\n    throw new RunRecordValidationError(`expected non-empty string`, path)\n  }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n  if (typeof value !== 'number' || !Number.isFinite(value)) {\n    throw new RunRecordValidationError(`expected finite number`, path)\n  }\n}\n\n/**\n * Heuristic snapshot check. Accepts:\n *   - `name@YYYY-MM-DD` (Anthropic style: `claude-sonnet-4-6@2025-04-15`)\n *   - `name-YYYYMMDD`   (OpenAI style: `gpt-4o-2024-11-20`)\n *   - `name@<arbitrary-token>` (allow opaque snapshots like `@v3`)\n *   - explicit `:date-...` Vertex-style tags\n *\n * Rejects bare aliases like `claude-sonnet-4` or `gpt-4o` that remap\n * silently as providers ship new snapshots.\n */\nfunction modelHasSnapshot(model: string): boolean {\n  if (model.includes('@')) return true\n  if (/-\\d{8}$/.test(model)) return true\n  if (/-\\d{4}-\\d{2}-\\d{2}$/.test(model)) return true\n  if (/:date-/.test(model)) return true\n  return false\n}\n"],"mappings":";AAsHA,IAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,aAAyC,CAAC,UAAU,OAAO,SAAS;AAEnE,IAAM,2BAAN,cAAuC,MAAM;AAAA,EACzC;AAAA,EACT,YAAY,SAAiB,OAAO,IAAI;AACtC,UAAM,OAAO,GAAG,OAAO,QAAQ,IAAI,MAAM,OAAO;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AACF;AAOO,SAAS,kBAAkB,OAA2B;AAC3D,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,IAAI,yBAAyB,iBAAiB;AAAA,EACtD;AACA,QAAM,MAAM;AAEZ,aAAW,OAAO,qBAAqB;AACrC,QAAI,EAAE,OAAO,MAAM;AACjB,YAAM,IAAI,yBAAyB,4BAA4B,GAAG,GAAG;AAAA,IACvE;AAAA,EACF;AAEA,eAAa,IAAI,OAAO,OAAO;AAC/B,eAAa,IAAI,cAAc,cAAc;AAC7C,eAAa,IAAI,aAAa,aAAa;AAC3C,qBAAmB,IAAI,MAAM,MAAM;AACnC,eAAa,IAAI,OAAO,OAAO;AAC/B,eAAa,IAAI,YAAY,YAAY;AACzC,eAAa,IAAI,YAAY,YAAY;AACzC,eAAa,IAAI,WAAW,WAAW;AACvC,qBAAmB,IAAI,QAAQ,QAAQ;AACvC,MAAI,IAAI,YAAY,OAAW,oBAAmB,IAAI,SAAS,SAAS;AACxE,qBAAmB,IAAI,SAAS,SAAS;AAGzC,MAAI,CAAC,iBAAiB,IAAI,KAAe,GAAG;AAC1C,UAAM,IAAI;AAAA,MACR,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAGA,QAAM,KAAK,IAAI;AACf,MAAI,OAAO,QAAQ,OAAO,OAAO,UAAU;AACzC,UAAM,IAAI,yBAAyB,gCAAgC,YAAY;AAAA,EACjF;AACA,QAAM,QAAQ;AACd,qBAAmB,MAAM,OAAO,kBAAkB;AAClD,qBAAmB,MAAM,QAAQ,mBAAmB;AACpD,MAAI,MAAM,WAAW,OAAW,oBAAmB,MAAM,QAAQ,mBAAmB;AAGpF,MAAI,IAAI,kBAAkB,QAAW;AACnC,UAAM,KAAK,IAAI;AACf,QAAI,OAAO,QAAQ,OAAO,OAAO,UAAU;AACzC,YAAM,IAAI,yBAAyB,mCAAmC,eAAe;AAAA,IACvF;AACA,UAAM,QAAQ;AACd,iBAAa,MAAM,OAAO,qBAAqB;AAC/C,iBAAa,MAAM,eAAe,6BAA6B;AAC/D,uBAAmB,MAAM,YAAY,0BAA0B;AAC/D,QAAI,OAAO,MAAM,aAAa,WAAW;AACvC,YAAM,IAAI,yBAAyB,0CAA0C,wBAAwB;AAAA,IACvG;AAAA,EACF;AAGA,QAAM,MAAM,IAAI;AAChB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,UAAU;AAC3C,UAAM,IAAI,yBAAyB,6BAA6B,SAAS;AAAA,EAC3E;AACA,QAAM,SAAS;AACf,MAAI,OAAO,gBAAgB,OAAW,oBAAmB,OAAO,aAAa,qBAAqB;AAClG,MAAI,OAAO,iBAAiB,OAAW,oBAAmB,OAAO,cAAc,sBAAsB;AACrG,MAAI,OAAO,gBAAgB,UAAa,OAAO,iBAAiB,QAAW;AACzE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,QAAM,MAAM,OAAO;AACnB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,UAAU;AAC3C,UAAM,IAAI,yBAAyB,iCAAiC,aAAa;AAAA,EACnF;AACA,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAA8B,GAAG;AACnE,uBAAmB,GAAG,eAAe,CAAC,EAAE;AAAA,EAC1C;AAGA,MAAI,IAAI,gBAAgB,OAAW,cAAa,IAAI,aAAa,aAAa;AAG9E,MAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GAAG;AACzF,UAAM,IAAI;AAAA,MACR,2BAA2B,WAAW,KAAK,IAAI,CAAC,SAAS,OAAO,IAAI,QAAQ,CAAC;AAAA,MAC7E;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAGO,SAAS,YAAY,OAAoC;AAC9D,MAAI;AACF,sBAAkB,KAAK;AACvB,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,mBACd,OAGiD;AACjD,MAAI;AACF,WAAO,EAAE,IAAI,MAAM,OAAO,kBAAkB,KAAK,EAAE;AAAA,EACrD,SAAS,GAAG;AACV,QAAI,aAAa,yBAA0B,QAAO,EAAE,IAAI,OAAO,OAAO,EAAE;AACxE,UAAM;AAAA,EACR;AACF;AAGO,SAAS,mBAAmB,QAA8B;AAC/D,QAAM,OAAO,KAAK,UAAU,MAAM;AAClC,SAAO,kBAAkB,KAAK,MAAM,IAAI,CAAC;AAC3C;AAIA,SAAS,aAAa,OAAgB,MAAoB;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAAA,EACtE;AACF;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;AAC9D,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACxD,UAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAAA,EACnE;AACF;AAYA,SAAS,iBAAiB,OAAwB;AAChD,MAAI,MAAM,SAAS,GAAG,EAAG,QAAO;AAChC,MAAI,UAAU,KAAK,KAAK,EAAG,QAAO;AAClC,MAAI,sBAAsB,KAAK,KAAK,EAAG,QAAO;AAC9C,MAAI,SAAS,KAAK,KAAK,EAAG,QAAO;AACjC,SAAO;AACT;","names":[]}

package/dist/cli.js

@@ -5,8 +5,9 @@ import {
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "./chunk-LSR4IAYN.js";
-import "./chunk-JAOLXRIA.js";
+} from "./chunk-WOPGKVN4.js";
+import "./chunk-3GN6U53I.js";
+import "./chunk-SNUHRBDL.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/cli.ts

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\n\nimport { buildOpenApi } from './wire/openapi'\nimport { handleVersion } from './wire/handlers'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServer } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]\n    if (tok.startsWith('--')) {\n      const key = tok.slice(2)\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval — wire-protocol entry point.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Method ∈ {judge, listRubrics, version}.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  version\n        Print server + wire-protocol version JSON.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n\n  switch (command) {\n    case 'serve': {\n      const port = Number(flags.port ?? 5005)\n      const host = flags.host ?? '127.0.0.1'\n      const server = startServer({ port, host })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      return await runRpcOnce(method)\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      return await runRpcBatch(method)\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, JSON.stringify(spec, null, 2) + '\\n', 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'version': {\n      process.stdout.write(JSON.stringify(handleVersion(), null, 2) + '\\n')\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(HELP + '\\n')\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n"],"mappings":";;;;;;;;;;;;AAaA,SAAS,qBAAqB;AAa9B,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAEtE,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,OAAO,MAAM,QAAQ,IAAI;AACtC,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,SAAS,YAAY,EAAE,MAAM,KAAK,CAAC;AAEzC,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,WAAW,MAAM;AAAA,IAChC;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,YAAY,MAAM;AAAA,IACjC;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,KAAK,UAAU,MAAM,MAAM,CAAC,IAAI,MAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,IAAI,IAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,OAAO,IAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n *   agent-eval serve [--port 5005] [--host 127.0.0.1]\n *   agent-eval rpc <method>          # one request from stdin → one response on stdout\n *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout\n *   agent-eval openapi [--out path]  # write OpenAPI spec\n *   agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\n\nimport { buildOpenApi } from './wire/openapi'\nimport { handleVersion } from './wire/handlers'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServer } from './wire/server'\n\ninterface Args {\n  command: string\n  positional: string[]\n  flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n  const [command, ...rest] = argv\n  const positional: string[] = []\n  const flags: Record<string, string> = {}\n  for (let i = 0; i < rest.length; i++) {\n    const tok = rest[i]\n    if (tok.startsWith('--')) {\n      const key = tok.slice(2)\n      const next = rest[i + 1]\n      if (next != null && !next.startsWith('--')) {\n        flags[key] = next\n        i++\n      } else {\n        flags[key] = 'true'\n      }\n    } else {\n      positional.push(tok)\n    }\n  }\n  return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval — wire-protocol entry point.\n\nCommands:\n  serve [--port 5005] [--host 127.0.0.1]\n        Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n  rpc <method>\n        Read one JSON object from stdin (the params for <method>), write one\n        JSON object to stdout. Method ∈ {judge, listRubrics, version}.\n  rpc-batch <method>\n        Like 'rpc' but JSONL in / JSONL out.\n  openapi [--out openapi.json]\n        Write the OpenAPI 3.1 spec.\n  version\n        Print server + wire-protocol version JSON.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n  const { command, positional, flags } = parseArgs(process.argv.slice(2))\n\n  switch (command) {\n    case 'serve': {\n      const port = Number(flags.port ?? 5005)\n      const host = flags.host ?? '127.0.0.1'\n      const server = startServer({ port, host })\n      // Keep process alive on SIGINT/SIGTERM\n      const shutdown = (sig: string) => {\n        // eslint-disable-next-line no-console\n        console.log(`[agent-eval] received ${sig}, shutting down`)\n        server.close(() => process.exit(0))\n        // Force exit after 5s if close hangs\n        setTimeout(() => process.exit(1), 5000).unref()\n      }\n      process.on('SIGINT', () => shutdown('SIGINT'))\n      process.on('SIGTERM', () => shutdown('SIGTERM'))\n      // Block forever\n      await new Promise(() => {})\n      return 0\n    }\n    case 'rpc': {\n      const [method] = positional\n      return await runRpcOnce(method)\n    }\n    case 'rpc-batch': {\n      const [method] = positional\n      return await runRpcBatch(method)\n    }\n    case 'openapi': {\n      const out = flags.out ?? 'openapi.json'\n      const spec = buildOpenApi(handleVersion().version)\n      writeFileSync(out, JSON.stringify(spec, null, 2) + '\\n', 'utf-8')\n      // eslint-disable-next-line no-console\n      console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n      return 0\n    }\n    case 'version': {\n      process.stdout.write(JSON.stringify(handleVersion(), null, 2) + '\\n')\n      return 0\n    }\n    case 'help':\n    case '--help':\n    case '-h':\n    case '':\n      process.stdout.write(HELP + '\\n')\n      return 0\n    default:\n      process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n      return 1\n  }\n}\n\nmain()\n  .then((code) => process.exit(code))\n  .catch((err) => {\n    // eslint-disable-next-line no-console\n    console.error('[agent-eval] cli error:', err)\n    process.exit(1)\n  })\n"],"mappings":";;;;;;;;;;;;;AAaA,SAAS,qBAAqB;AAa9B,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAEtE,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,OAAO,MAAM,QAAQ,IAAI;AACtC,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,SAAS,YAAY,EAAE,MAAM,KAAK,CAAC;AAEzC,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,WAAW,MAAM;AAAA,IAChC;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,YAAY,MAAM;AAAA,IACjC;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,KAAK,UAAU,MAAM,MAAM,CAAC,IAAI,MAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,IAAI,IAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,OAAO,IAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/control-cxwMOAsy.d.ts

@@ -0,0 +1,259 @@
+import { c as ControlEvalResult, i as ControlRunResult, F as FeedbackLabel, A as ProposedSideEffect, j as ControlRuntimeConfig } from './feedback-trajectory-CB0A32o3.js';
+import { R as RunSplitTag, e as RunTokenUsage, a as RunRecord } from './run-record-CX_jcAyr.js';
+import { T as TraceStore, F as FailureClass } from './store-u47QaJ9G.js';
+import { T as TraceEmitter } from './emitter-B2XqDKFU.js';
+
+interface RunEvidenceMetadata {
+    experimentId: string;
+    candidateId: string;
+    seed: number;
+    model: string;
+    promptHash: string;
+    configHash: string;
+    commitSha: string;
+    splitTag: RunSplitTag;
+    tokenUsage: RunTokenUsage;
+    queueMs?: number;
+    judgeMetadata?: RunRecord['judgeMetadata'];
+    raw?: Record<string, number>;
+}
+interface ControlRunToRunRecordOptions extends RunEvidenceMetadata {
+    runId?: string;
+    score?: number;
+    failureMode?: string;
+}
+/**
+ * Project a completed control-loop run into the strict RunRecord shape used by
+ * release gates, optimizer tables, and research reports.
+ *
+ * The control loop owns live execution evidence. The caller still supplies the
+ * experimental cell metadata because prompt/config hashes, split assignment,
+ * model snapshot, and commit SHA are product/harness concerns.
+ */
+declare function controlRunToRunRecord<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(run: ControlRunResult<TState, TAction, TActionResult, TEval>, options: ControlRunToRunRecordOptions): RunRecord;
+declare function scoreFromEvals(evals: readonly ControlEvalResult[]): number | undefined;
+
+interface ActionExecutionPolicy {
+    allowedTypes?: string[];
+    blockedTypes?: string[];
+    alwaysRequireApprovalTypes?: string[];
+    autoApproveTypes?: string[];
+    requireApprovalForExternalSideEffects?: boolean;
+    requireApprovalAboveCostUsd?: number;
+    maxActionCostUsd?: number;
+    remainingBudgetUsd?: number;
+    expectedOutcomeRequired?: boolean;
+    killCriteriaRequired?: boolean;
+}
+interface ActionPolicyDecision {
+    allowed: boolean;
+    blocked: boolean;
+    requiresApproval: boolean;
+    reasons: string[];
+    label?: FeedbackLabel;
+}
+declare function evaluateActionPolicy(action: ProposedSideEffect, policy?: ActionExecutionPolicy, options?: {
+    createdAt?: string;
+}): ActionPolicyDecision;
+
+/**
+ * Propose / Verify / Review — the core multi-shot primitive.
+ *
+ *   shot N:  propose(state, priorReview) → new state
+ *            verify(state)               → pass/fail, optional layers
+ *            review(state, verification, memory) → observations + next-shot
+ *                                                   instruction + shouldContinue
+ *            memory.append(entry)
+ *
+ * Roles are strictly separated:
+ *
+ *   - The WORKER is whatever the caller wraps in `propose`. It is
+ *     stateful — caller owns its resume/session mechanism.
+ *   - The VERIFIER grades the state. It produces the ground truth.
+ *     The reviewer cannot overturn or downgrade a verification layer.
+ *   - The REVIEWER is stateless per call. Its continuity is the
+ *     `ReviewMemoryStore` — durable JSONL by default, or any store
+ *     implementing the interface. It reads memory + trace summary +
+ *     verification and directs the NEXT proposer shot.
+ *
+ * This shape is load-bearing. The reviewer never grades; the verifier
+ * never directs. Two processes, two prompts, two concerns — which is
+ * what keeps the loop from confirmation-biasing itself into "all
+ * passed" when it didn't.
+ *
+ * Short-circuits and soft-fails are both first-class:
+ *   - verify.pass === true  → reviewer LLM call is skipped, memory
+ *     records a success entry, loop exits.
+ *   - review throws         → the shot still counts; the loop uses the
+ *     last-known instruction (or `fallbackInstruction`) for the next
+ *     propose call. A transient reviewer failure must NEVER abort a
+ *     valid arc.
+ *
+ * Composable: `propose` itself can be another `runProposeReview` call.
+ * That's the dogfooding path — a harness built on this primitive is in
+ * turn evaluable by it.
+ */
+
+interface Verification {
+    pass: boolean;
+    score?: number;
+    failingLayers?: string[];
+    details?: unknown;
+}
+interface Review {
+    observations: string;
+    diagnosis: string;
+    nextShotInstruction: string;
+    shouldContinue: boolean;
+    confidence: number;
+}
+interface ReviewMemoryEntry extends Review {
+    shot: number;
+    timestamp: number;
+    verification: {
+        pass: boolean;
+        score?: number;
+        failingLayers?: string[];
+    };
+}
+interface ProposeInput<State> {
+    shot: number;
+    goal: string;
+    state: State;
+    priorReview: Review | null;
+    abortSignal: AbortSignal;
+    emitter?: TraceEmitter;
+}
+interface ProposeOutput<State, Summary = unknown> {
+    state: State;
+    traceSummary?: Summary;
+}
+interface ReviewInput<State, Summary = unknown> {
+    shot: number;
+    goal: string;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    memory: ReviewMemoryEntry[];
+}
+type ProposeFn<State, Summary = unknown> = (input: ProposeInput<State>) => Promise<ProposeOutput<State, Summary>>;
+type VerifyFn<State> = (state: State) => Promise<Verification>;
+type ReviewFn<State, Summary = unknown> = (input: ReviewInput<State, Summary>) => Promise<Review>;
+interface ReviewMemoryStore {
+    load(): Promise<ReviewMemoryEntry[]>;
+    append(entry: ReviewMemoryEntry): Promise<void>;
+}
+interface ProposeReviewConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    /** Hard shot cap. Default 10. */
+    maxShots?: number;
+    /** Wall-clock cap in ms. Default 10 min. */
+    maxWallMs?: number;
+    /**
+     * If the reviewer returns confidence ≤ floor on `confidenceFloorWindow`
+     * consecutive shots, terminate early. Default floor 0.3, window 2.
+     * Set window to 0 or floor to <0 to disable.
+     */
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    /** Defaults to an in-memory store if omitted. */
+    memory?: ReviewMemoryStore;
+    /** If provided, emit a Run + per-shot spans. */
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    /**
+     * Used when the reviewer soft-fails on shot 1 (no prior instruction to
+     * fall back to). Default is a generic "inspect failures and fix".
+     */
+    fallbackInstruction?: string;
+}
+interface ProposeReviewShot<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    review: Review;
+    reviewAvailable: boolean;
+    reviewError?: string;
+    durationMs: number;
+}
+interface ProposeReviewReport<State, Summary = unknown> {
+    runId: string | null;
+    completed: boolean;
+    shots: ProposeReviewShot<State, Summary>[];
+    finalState: State;
+    finalVerification: Verification;
+    failureClass?: FailureClass;
+    wallMs: number;
+    score: number;
+}
+declare function inMemoryReviewStore(initial?: ReviewMemoryEntry[]): ReviewMemoryStore;
+declare function jsonlReviewStore(path: string): ReviewMemoryStore;
+declare function runProposeReview<State, Summary = unknown>(config: ProposeReviewConfig<State, Summary>): Promise<ProposeReviewReport<State, Summary>>;
+interface LlmJsonCall {
+    (req: {
+        system: string;
+        user: string;
+    }): Promise<unknown>;
+}
+interface LlmReviewerConfig<State, Summary = unknown> {
+    callJson: LlmJsonCall;
+    renderState?: (state: State) => string;
+    renderTraceSummary?: (summary: Summary | undefined) => string;
+    /** Appended to the default system prompt. */
+    systemPromptAddendum?: string;
+}
+declare function createLlmReviewer<State, Summary = unknown>(cfg: LlmReviewerConfig<State, Summary>): ReviewFn<State, Summary>;
+
+interface ProposeReviewControlState<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    priorReview: Review | null;
+    verification: Verification;
+    traceSummary?: Summary;
+    memory: ReviewMemoryEntry[];
+    completed: boolean;
+    reviewAvailable: boolean;
+    reviewError?: string;
+}
+interface ProposeReviewControlAction {
+    type: 'propose-review-shot';
+    shot: number;
+}
+interface ProposeReviewControlResult<State, Summary = unknown> {
+    state: State;
+    verification: Verification;
+    traceSummary?: Summary;
+    review: Review | null;
+    reviewAvailable: boolean;
+    reviewError?: string;
+}
+interface ProposeReviewControlConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    maxShots?: number;
+    maxWallMs?: number;
+    memory?: ReviewMemoryStore;
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    fallbackInstruction?: string;
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    failureClassFromVerification?: (verification: Verification) => FailureClass | undefined;
+    actionFailure?: ControlRuntimeConfig<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>['actionFailure'];
+}
+declare function runProposeReviewAsControlLoop<State, Summary = unknown>(config: ProposeReviewControlConfig<State, Summary>): Promise<ControlRunResult<ProposeReviewControlState<State, Summary>, ProposeReviewControlAction, ProposeReviewControlResult<State, Summary>>>;
+declare function controlFailureClassFromVerification(verification: Verification): FailureClass | undefined;
+
+export { type ActionExecutionPolicy as A, type ControlRunToRunRecordOptions as C, type LlmJsonCall as L, type ProposeFn as P, type Review as R, type Verification as V, type ActionPolicyDecision as a, type LlmReviewerConfig as b, type ProposeInput as c, type ProposeOutput as d, type ProposeReviewConfig as e, type ProposeReviewControlAction as f, type ProposeReviewControlConfig as g, type ProposeReviewControlResult as h, type ProposeReviewControlState as i, type ProposeReviewReport as j, type ProposeReviewShot as k, type ReviewFn as l, type ReviewInput as m, type ReviewMemoryEntry as n, type ReviewMemoryStore as o, type RunEvidenceMetadata as p, type VerifyFn as q, controlFailureClassFromVerification as r, controlRunToRunRecord as s, createLlmReviewer as t, evaluateActionPolicy as u, inMemoryReviewStore as v, jsonlReviewStore as w, runProposeReview as x, runProposeReviewAsControlLoop as y, scoreFromEvals as z };

package/dist/control.d.ts

@@ -0,0 +1,6 @@
+export { d as ControlActionFailureMode, e as ControlActionOutcome, f as ControlBudget, g as ControlContext, h as ControlDecision, c as ControlEvalResult, i as ControlRunResult, j as ControlRuntimeConfig, k as ControlRuntimeError, C as ControlSeverity, l as ControlStep, m as ControlStopPolicies, S as StopDecision, B as allCriticalPassed, M as objectiveEval, T as runAgentControlLoop, V as stopOnNoProgress, W as stopOnRepeatedAction, X as subjectiveEval } from './feedback-trajectory-CB0A32o3.js';
+export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, p as RunEvidenceMetadata, s as controlRunToRunRecord, u as evaluateActionPolicy, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-cxwMOAsy.js';
+import './dataset-B9qvlm_o.js';
+import './emitter-B2XqDKFU.js';
+import './store-u47QaJ9G.js';
+import './run-record-CX_jcAyr.js';

package/dist/control.js

@@ -0,0 +1,30 @@
+import {
+  allCriticalPassed,
+  controlRunToRunRecord,
+  evaluateActionPolicy,
+  objectiveEval,
+  runAgentControlLoop,
+  runProposeReview,
+  runProposeReviewAsControlLoop,
+  scoreFromEvals,
+  stopOnNoProgress,
+  stopOnRepeatedAction,
+  subjectiveEval
+} from "./chunk-ARZ6BEV6.js";
+import "./chunk-YUFXO3TU.js";
+import "./chunk-5IIQKMD5.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  allCriticalPassed,
+  controlRunToRunRecord,
+  evaluateActionPolicy,
+  objectiveEval,
+  runAgentControlLoop,
+  runProposeReview,
+  runProposeReviewAsControlLoop,
+  scoreFromEvals,
+  stopOnNoProgress,
+  stopOnRepeatedAction,
+  subjectiveEval
+};
+//# sourceMappingURL=control.js.map

package/dist/control.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/dataset-B9qvlm_o.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Dataset — versioned, sliceable, content-hashed scenario collection.
+ *
+ * Scenarios stop being ephemeral arrays and become first-class
+ * artifacts. Every Dataset carries:
+ *   - content hash (sha256 over canonicalized scenario array)
+ *   - provenance (contributor, createdAt, sourceUrl)
+ *   - split labels (train | dev | test | holdout)
+ *   - difficulty tiers (easy | medium | hard | extreme)
+ *   - tags (free-form, per-scenario)
+ *
+ * `Dataset.slice({ difficulty, split, holdout, seed })` returns a
+ * deterministic, reproducible subset. Holdout slices are locked: you
+ * can read them but `mutate` throws, which prevents "oh I'll just
+ * tweak that one scenario" contamination drift.
+ */
+type DatasetSplit = 'train' | 'dev' | 'test' | 'holdout';
+type DatasetDifficulty = 'easy' | 'medium' | 'hard' | 'extreme';
+interface DatasetScenario {
+    id: string;
+    /** Arbitrary payload; the framework doesn't interpret it. */
+    payload: unknown;
+    split?: DatasetSplit;
+    difficulty?: DatasetDifficulty;
+    /** Canary token that MUST NOT round-trip through a correct agent output. */
+    canary?: string;
+    /**
+     * Behavioral-canary forbidden pattern. A string OR a serialized regex
+     * (`/.../flags`) that the agent under test MUST NOT emit. Used by
+     * {@link import('./canary').checkBehavioralCanary | checkBehavioralCanary},
+     * which inverts the contamination-style semantic: presence in the
+     * agent output is a LEAK / failure, not a positive signal.
+     *
+     * Falls back to {@link canary} when omitted.
+     */
+    forbiddenPattern?: string;
+    tags?: Record<string, string>;
+}
+interface DatasetProvenance {
+    contributor?: string;
+    createdAt: string;
+    sourceUrl?: string;
+    license?: string;
+    description?: string;
+    /** Monotonic human-readable version (e.g. "2026.04.20"). */
+    version: string;
+}
+interface DatasetManifest {
+    name: string;
+    provenance: DatasetProvenance;
+    /** sha256 hex over canonicalized scenarios. */
+    contentHash: string;
+    scenarioCount: number;
+    splitCounts: Record<DatasetSplit, number>;
+}
+interface SliceOptions {
+    split?: DatasetSplit;
+    difficulty?: DatasetDifficulty;
+    /** Number of scenarios (random sample, seeded). Omit to take all that match. */
+    limit?: number;
+    seed?: number;
+    /** Predicate narrowing. Applied after split/difficulty filters. */
+    filter?: (scenario: DatasetScenario) => boolean;
+    /** If true, include scenarios marked as holdout. Default false. */
+    includeHoldout?: boolean;
+}
+/** Locked holdouts — throws on mutate. Callers that need a mutable dataset fork it. */
+declare class HoldoutLockedError extends Error {
+    constructor(datasetName: string);
+}
+declare class Dataset {
+    readonly name: string;
+    readonly provenance: DatasetProvenance;
+    private scenarios;
+    private locked;
+    constructor(init: {
+        name: string;
+        provenance: DatasetProvenance;
+        scenarios: DatasetScenario[];
+        locked?: boolean;
+    });
+    /** All scenarios. Readonly — callers must go through `slice` or `clone`. */
+    all(): readonly DatasetScenario[];
+    get size(): number;
+    /**
+     * Deterministic sliced subset. Seed is REQUIRED when `limit` is set so
+     * the same arguments always produce the same slice across machines.
+     */
+    slice(options?: SliceOptions): DatasetScenario[];
+    /**
+     * Assemble the manifest (name + provenance + content hash + counts).
+     * Content hash is deterministic over canonicalized scenarios.
+     */
+    manifest(): Promise<DatasetManifest>;
+    /** Fresh unlocked copy — for post-release forks when mutation is needed. */
+    clone(overrides?: Partial<{
+        name: string;
+        version: string;
+    }>): Dataset;
+    lock(): void;
+    add(scenario: DatasetScenario): void;
+    remove(scenarioId: string): void;
+    /**
+     * Stable JSON-Lines serialization — deterministic byte-for-byte.
+     * Write to disk for contamination-verifiable archives.
+     */
+    toJsonl(): string;
+    static fromJsonl(jsonl: string, manifest: Omit<DatasetManifest, 'contentHash' | 'scenarioCount' | 'splitCounts'>): Dataset;
+}
+declare function hashScenarios(scenarios: DatasetScenario[]): Promise<string>;
+
+export { type DatasetSplit as D, HoldoutLockedError as H, type SliceOptions as S, type DatasetScenario as a, Dataset as b, type DatasetManifest as c, type DatasetDifficulty as d, type DatasetProvenance as e, hashScenarios as h };

package/dist/emitter-B2XqDKFU.d.ts

@@ -0,0 +1,121 @@
+import { T as TraceStore, R as RunOutcome, a as Run, S as Span, b as SpanKind, L as LlmSpan, c as ToolSpan, d as RetrievalSpan, J as JudgeSpan, e as SandboxSpan, E as EventKind, f as TraceEvent, B as BudgetLedgerEntry, A as Artifact, M as Message } from './store-u47QaJ9G.js';
+
+/**
+ * TraceEmitter — hierarchical span builder that auto-parents using an
+ * internal stack. One emitter per Run; emitters do NOT share state.
+ *
+ * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
+ * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
+ * have to thread spanIds manually. For async workflows that can't use
+ * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
+ * explicitly.
+ */
+
+interface SpanHandle<S extends Span = Span> {
+    span: S;
+    end(patch?: Partial<S>): Promise<void>;
+    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
+}
+interface RunCompleteHookContext {
+    runId: string;
+    emitter: TraceEmitter;
+    store: TraceStore;
+    /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
+    outcome?: RunOutcome;
+    /** Final run status. */
+    status: 'completed' | 'failed' | 'aborted';
+}
+type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
+interface TraceEmitterOptions {
+    runId?: string;
+    /** Inject a clock for deterministic tests. */
+    now?: () => number;
+    /** Inject an id generator for deterministic tests. */
+    id?: () => string;
+    /**
+     * Hooks fired after `endRun` / `abortRun` writes the final run state.
+     * Designed for trace-analyst auto-execution, integrity assertions, and
+     * outbound notifications. Hooks run sequentially in the order supplied.
+     *
+     * By default a hook that throws is swallowed and logged as a `note` event
+     * on the run — auto-orchestration must not crash the underlying flow.
+     * Set `hookErrors: 'throw'` to propagate.
+     */
+    onRunComplete?: RunCompleteHook[];
+    /** `'swallow'` (default) | `'throw'`. */
+    hookErrors?: 'swallow' | 'throw';
+}
+declare class TraceEmitter {
+    private store;
+    private stack;
+    private _runId;
+    private now;
+    private id;
+    private hooks;
+    private hookErrors;
+    constructor(store: TraceStore, options?: TraceEmitterOptions);
+    get runId(): string;
+    get traceStore(): TraceStore;
+    /** Append a hook after construction (e.g. attach the trace analyst). */
+    addRunCompleteHook(hook: RunCompleteHook): void;
+    /**
+     * Begin a Run.
+     *
+     * `scenarioId` is required on the persisted Run shape — every Run downstream
+     * gets a non-empty scenarioId so filters and aggregations stay simple — but
+     * the INPUT here accepts it as optional. When omitted, startRun substitutes
+     * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
+     * runtime / operator / meta-eval runs that have no curated-scenario corpus
+     * to anchor to don't have to invent placeholder strings at the call site.
+     */
+    startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
+        scenarioId?: string;
+    }): Promise<Run>;
+    endRun(outcome?: RunOutcome): Promise<void>;
+    abortRun(reason: string): Promise<void>;
+    private runHooks;
+    span<S extends Span = Span>(init: {
+        kind: SpanKind;
+        name: string;
+        parentSpanId?: string;
+        attributes?: Record<string, unknown>;
+    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
+    private handle;
+    private pop;
+    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
+    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
+    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
+    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
+    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
+    emit(event: {
+        kind: EventKind;
+        spanId?: string;
+        payload?: Record<string, unknown>;
+    }): Promise<TraceEvent>;
+    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
+        timestamp?: number;
+    }): Promise<BudgetLedgerEntry>;
+    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
+    /**
+     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
+     * Returns the fn's return value. Use this for the 95% case.
+     */
+    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
+}
+/** Helper to build an LLM span handle args object from a provider-shaped response. */
+declare function llmSpanFromProvider(args: {
+    name?: string;
+    model: string;
+    messages: Message[];
+    output: string;
+    usage?: {
+        inputTokens?: number;
+        outputTokens?: number;
+        cachedTokens?: number;
+        reasoningTokens?: number;
+    };
+    costUsd?: number;
+    finishReason?: string;
+}): Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>;
+
+export { type RunCompleteHook as R, type SpanHandle as S, TraceEmitter as T, type RunCompleteHookContext as a, type TraceEmitterOptions as b, llmSpanFromProvider as l };