@runcofactor/cli 0.57.0 → 0.58.0

package/dist/{chunk-T5MU76RA.js → chunk-C77XIO2W.js}

@@ -3,16 +3,16 @@ import {
   apiRequest,
   getPositionalArgs,
   readStdin
-} from "./chunk-72Y65NWI.js";
+} from "./chunk-WY4R4IL5.js";
 import {
   GLOBAL_VALUE_FLAGS,
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-IIDQM4MY.js";
+} from "./chunk-BMFD4OPC.js";
 import {
   writeRawOutput
-} from "./chunk-T6ZNIMI4.js";
+} from "./chunk-LRIP5IQD.js";
 
 // src/utilities.ts
 import fs from "fs/promises";
@@ -499,4 +499,4 @@ export {
   parseDurationSeconds,
   runUtilities
 };
-//# sourceMappingURL=chunk-T5MU76RA.js.map
+//# sourceMappingURL=chunk-C77XIO2W.js.map

package/dist/{chunk-FBEQNW2P.js → chunk-DFN2NH37.js}

@@ -9,7 +9,7 @@ import {
   PRODUCT_MAP_STATUSES,
   WORKFLOW_FLOW_TYPES,
   WORKFLOW_MANAGEMENT_STATUSES
-} from "./chunk-EYHJ5FIA.js";
+} from "./chunk-C6SNLHVE.js";
 
 // ../mcp-tools/src/issue-evidence.ts
 var MEDIA_FIELDS = {
@@ -5508,4 +5508,4 @@ export {
   resolveTeamsChannelRefs,
   sharedGroups
 };
-//# sourceMappingURL=chunk-FBEQNW2P.js.map
+//# sourceMappingURL=chunk-DFN2NH37.js.map

package/dist/{chunk-4SQKKD3R.js → chunk-EDX47DXT.js}

@@ -9,7 +9,7 @@ import {
   slugifySeedName,
   validateMacroReferences,
   validateWorkflowInputExpressions
-} from "./chunk-EYHJ5FIA.js";
+} from "./chunk-C6SNLHVE.js";
 
 // ../canary-spec/src/parse.ts
 import yaml2 from "yaml";
@@ -3256,4 +3256,4 @@ export {
   writeWorkflowId,
   writeNodeIds
 };
-//# sourceMappingURL=chunk-4SQKKD3R.js.map
+//# sourceMappingURL=chunk-EDX47DXT.js.map

package/dist/{chunk-BSCEOIYX.js → chunk-FQ6P522C.js}

@@ -1,7 +1,7 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
 import {
   command
-} from "./chunk-T6ZNIMI4.js";
+} from "./chunk-LRIP5IQD.js";
 
 // src/teardown-summary.ts
 var ICONS = {
@@ -35,4 +35,4 @@ function formatTeardownLines(teardowns) {
 export {
   formatTeardownLines
 };
-//# sourceMappingURL=chunk-BSCEOIYX.js.map
+//# sourceMappingURL=chunk-FQ6P522C.js.map

package/dist/{chunk-6IMNRYC3.js → chunk-GQSW6ZY3.js}

@@ -2,10 +2,10 @@ import { createRequire as __cr } from "module"; import { fileURLToPath as __fup
 import {
   resolveIsSuperadmin,
   showGlobalHelp
-} from "./chunk-DTB5HUD2.js";
+} from "./chunk-SPTLZJOF.js";
 import {
   apiRequest
-} from "./chunk-72Y65NWI.js";
+} from "./chunk-WY4R4IL5.js";
 import {
   getSubDir
 } from "./chunk-S3Q7MDCS.js";
@@ -14,10 +14,10 @@ import {
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-IIDQM4MY.js";
+} from "./chunk-BMFD4OPC.js";
 import {
   writeRawOutput
-} from "./chunk-T6ZNIMI4.js";
+} from "./chunk-LRIP5IQD.js";
 
 // src/landing.ts
 import { readFile, writeFile, mkdir } from "fs/promises";
@@ -623,4 +623,4 @@ export {
   buildLandingUrl,
   runLanding
 };
-//# sourceMappingURL=chunk-6IMNRYC3.js.map
+//# sourceMappingURL=chunk-GQSW6ZY3.js.map

package/dist/{chunk-4GDEKB5S.js → chunk-GVNSCSIC.js}

@@ -1,16 +1,16 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
 import {
   apiRequest
-} from "./chunk-72Y65NWI.js";
+} from "./chunk-WY4R4IL5.js";
 import {
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-IIDQM4MY.js";
+} from "./chunk-BMFD4OPC.js";
 import {
   writeRawOutput,
   writeUntrustedText
-} from "./chunk-T6ZNIMI4.js";
+} from "./chunk-LRIP5IQD.js";
 
 // src/triage.ts
 import process from "process";
@@ -274,4 +274,4 @@ export {
   printTriageSummary,
   runTriage
 };
-//# sourceMappingURL=chunk-4GDEKB5S.js.map
+//# sourceMappingURL=chunk-GVNSCSIC.js.map

package/dist/{chunk-W4P6BGJ7.js → chunk-IN3AF6LI.js}

@@ -3,21 +3,21 @@ import {
   captureSubstrateV2,
   parseRunCodeStrict,
   parseSnapshot
-} from "./chunk-CZOYFKTY.js";
+} from "./chunk-IVJU3LP6.js";
 import {
   consoleLogger
 } from "./chunk-5QHORHJT.js";
 import {
   reconstructSnapshots,
   toMutationIndex
-} from "./chunk-LP4X7TWI.js";
+} from "./chunk-56WV7CFM.js";
 import {
   MACRO_PATTERN,
   cacheActionContractIssues,
   isVolatileIdentifier,
   stableAttrSelector,
   stableIdentifierPrefix
-} from "./chunk-EYHJ5FIA.js";
+} from "./chunk-C6SNLHVE.js";
 
 // ../workflow-cache/src/logger.ts
 var activeLogger = consoleLogger;
@@ -9779,4 +9779,4 @@ export {
   judgeNodeReplayProof,
   judgeCacheOnlyRunClean
 };
-//# sourceMappingURL=chunk-W4P6BGJ7.js.map
+//# sourceMappingURL=chunk-IN3AF6LI.js.map

package/dist/{chunk-CZOYFKTY.js → chunk-IVJU3LP6.js}

@@ -10,7 +10,7 @@ import {
 } from "./chunk-5QHORHJT.js";
 import {
   redactTraceArchive
-} from "./chunk-LP4X7TWI.js";
+} from "./chunk-56WV7CFM.js";
 import {
   BROWSER_VIEWPORT_PRESETS,
   DEFAULT_BROWSER_ENGINE,
@@ -20,7 +20,7 @@ import {
   isVolatileIdentifier,
   redactSecretsWithCount,
   sensitiveNeedles
-} from "./chunk-EYHJ5FIA.js";
+} from "./chunk-C6SNLHVE.js";
 import {
   __commonJS,
   __esm,
@@ -310834,4 +310834,4 @@ playwright-extra/dist/index.esm.js:
    * @license MIT
    *)
 */
-//# sourceMappingURL=chunk-CZOYFKTY.js.map
+//# sourceMappingURL=chunk-IVJU3LP6.js.map

package/dist/{chunk-H2K3ABPO.js → chunk-K54XL64A.js}

@@ -1,17 +1,17 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
 import {
   formatSemanticSnapshot
-} from "./chunk-CZOYFKTY.js";
+} from "./chunk-IVJU3LP6.js";
 import {
   getPositionalArgs
-} from "./chunk-72Y65NWI.js";
+} from "./chunk-WY4R4IL5.js";
 import {
   GLOBAL_VALUE_FLAGS,
   getArgValue
-} from "./chunk-IIDQM4MY.js";
+} from "./chunk-BMFD4OPC.js";
 import {
   writeRawOutput
-} from "./chunk-T6ZNIMI4.js";
+} from "./chunk-LRIP5IQD.js";
 
 // src/trace.ts
 import fs from "fs";
@@ -62,7 +62,7 @@ async function runTraceSnapshot(argv) {
     process.exit(1);
   }
   const filter = filterArg;
-  const { reconstructSnapshots, parseTraceArchive, downloadTrace, cleanupTrace } = await import("./src-DJETLMXP.js");
+  const { reconstructSnapshots, parseTraceArchive, downloadTrace, cleanupTrace } = await import("./src-BCJZMHSQ.js");
   let localPath = tracePath;
   let isTemp = false;
   if (tracePath.startsWith("http://") || tracePath.startsWith("https://")) {
@@ -183,4 +183,4 @@ function slugify(text) {
 export {
   runTrace
 };
-//# sourceMappingURL=chunk-H2K3ABPO.js.map
+//# sourceMappingURL=chunk-K54XL64A.js.map

package/dist/{chunk-T6ZNIMI4.js → chunk-LRIP5IQD.js}

@@ -1613,7 +1613,7 @@ var SANDBOX_GUIDE_SECTIONS = [
 ];
 
 // ../content/src/external-run-guide/overview.txt
-var overview_default = "External-run diagnosis is a customer-accessible workflow. It requires a normal\nCofactor login with external-run read access; it does not require psql, Sentry,\nAWS, Redis, or superadmin commands.\n\nStart with:\n\n  cofactor external-run get <run-id>\n  cofactor external-run failures <run-id> [--area <label|index>]\n  cofactor external-run triage <run-id> [--area <label|index>]\n  cofactor external-run skips <run-id>\n  cofactor external-run debug overview <run-id>\n\nUse the result rows to answer what the test framework observed. Use lifecycle\ntelemetry to answer whether the harness timed out, retried, or recovered. Use\nartifacts and a focused rerun to distinguish an application bug from a test or\nharness issue.\n\nCofactor's own AI findings are readable too: `triage` groups a run's failures\ninto ranked findings, and `diagnosis --item` shows one test's root-cause\ndiagnosis (see the `ai-findings` section). Treat both as leads to verify\nagainst the evidence, not as conclusions. A developer or local coding agent can\ninspect the materialized workspace alongside the customer's test source and\napplication source.\n";
+var overview_default = "External-run diagnosis is a customer-accessible workflow. It requires a normal\nCofactor login with external-run read access; it does not require psql, Sentry,\nAWS, Redis, or superadmin commands.\n\nStart with:\n\n  cofactor external-run comments <run-id> --item <item-id>\n  cofactor external-run comment <run-id> --item <item-id> --body \"Setup failed\" --org Acme\n  cofactor external-run comments --test <test-id> --json\n  cofactor external-run comment --test <test-id> --body \"Waiting for a fix\" --org Acme\n  cofactor external-run get <run-id>\n  cofactor external-run failures <run-id> [--area <label|index>]\n  cofactor external-run triage <run-id> [--area <label|index>]\n  cofactor external-run skips <run-id>\n  cofactor external-run debug overview <run-id>\n\nUse the result rows to answer what the test framework observed. Use lifecycle\ntelemetry to answer whether the harness timed out, retried, or recovered. Use\nartifacts and a focused rerun to distinguish an application bug from a test or\nharness issue.\n\nCofactor's own AI findings are readable too: `triage` groups a run's failures\ninto ranked findings, and `diagnosis --item` shows one test's root-cause\ndiagnosis (see the `ai-findings` section). Treat both as leads to verify\nagainst the evidence, not as conclusions. A developer or local coding agent can\ninspect the materialized workspace alongside the customer's test source and\napplication source.\n\nExternal test comments follow the test across runs, newest first, with author and time.\nA run-item comment defaults to that result and includes its run link; pass `--about test`\nto leave a general note. Bodies accept text or @file (1\u201320000 characters).\nRead comments with external-test-comment:read; add them with external-test-comment:create.\nItem targets additionally need external-test-run:read to resolve the durable test.\nLists accept --page (1\u201310000) and --page-size (1\u2013200, default 25); --json includes pagination.\nOut-of-range paging values are rejected. --about accepts test or run only when\nadding a comment addressed by a run item. A denied item lookup explains that\nexternal-test-run:read is needed; use --test to address the thread directly instead.\nThe only comment route is /external-tests/tests/:testId/comments; item detail exposes\nexternalTestId for resolving the thread. POST accepts { body, runItemId? }; omitting\nrunItemId leaves a general note. GET returns { ok, data: comments, pagination };\nComments contain { id, runItemId, authorId, authorName, body, createdAt, run };\nrun is null or { runId, createdAt }. Clients use runItemId to build the result link.\n";
 
 // ../content/src/external-run-guide/materialize.txt
 var materialize_default = "Create a bounded local workspace:\n\n  cofactor external-run debug materialize <run-id>\n\nThe default downloads run logs plus first-failure screenshots and DOM evidence\nfor failed/flaky scenarios. Video is excluded by default because it is large:\n\n  --include-video\n  --artifacts none|failures|all\n  --out <directory>\n\nThe output directory must be new or empty; the command refuses to overwrite an\nexisting workspace.\n\nNarrow the workspace when one scenario or one area is under investigation:\n\n  cofactor external-run debug materialize <run-id> --item <item-id-or-text>\n  cofactor external-run debug materialize <run-id> --area <label|index>\n\nEvery selected scenario with a Playwright trace gets a full diagnostic\nworkspace under traces/<scenario>/: timeline/test-steps.jsonl (the test's step\ntree with times, source locations and the failing step's error),\nbrowser/actions.jsonl with a DOM, screenshot and accessibility snapshot after\neach call, browser/error-context.md (Playwright's page snapshot at the failure),\nlogs/test-output.jsonl (stdout/stderr), source/, and the raw browser/trace.zip.\n\nFor one test's trace alone:\n\n  cofactor external-run trace <run-id> --item <item-id-or-text> [--out <dir>]\n\nOnly retried attempts are traced unless the suite records every attempt, so a\ntest that failed without retrying has no trace; work from its error, steps,\nlogs and video.\n\nThe workspace contains summary.md, manifest.json, JSONL result rows, folded\nscenarios, skip explanations, lifecycle telemetry, scenario histories,\ndiagnostics, and an artifact manifest. Downloads are best-effort: a retained row\nwhose artifact has expired is recorded as unavailable instead of aborting the\nwhole materialization.\n\nA scenario whose test reported its own stdout, stderr, or a text attachment\n(tags and source location included) gets output/<scenario>.txt \u2014 Playwright's\nprimary-source capture of what the test itself printed, distinct from the\ncontainer's run.log under logs/.\n";
@@ -1917,7 +1917,7 @@ var PERF_GUIDE_SECTIONS = [
 ];
 
 // ../content/src/eval-guide/overview.txt
-var overview_default6 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 1                      # attempts per case, 1 to 7 (default 1; for a stochastic agent raise it with passAtLeast)\n  passAtLeast: 1                  # a case passes when this many attempts pass (default 1, at most repeats)\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n  EVAL_TRACE_CAPTURE   full (default) or metadata\n  EVAL_NODE            the runner\'s trusted Node binary\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n[cli >= 0.56.0]\n- A wrong label or input is corrected with a new version, not by deleting the\n  case: `cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n  --expected <file|-> --source "<provenance>" --replace --reason "<why>" --org <org>`\n  stores version N+1 with who, when and the reason. Without `--replace` an\n  existing key is still an error. Runs that already used the case keep the\n  version they graded and show it as `key (vN)`; new runs use the latest.\n  `cofactor eval compare` refuses runs that graded different case versions, so\n  re-run the baseline after a correction.\n- `cofactor eval case <name> <key> [--version N]` prints a case\'s expected\n  label, provenance and version history (newest first).\n[/cli]\n\nCASE FIDELITY AND EVIDENCE MAP (REQUIRED)\n\nThe label is verified truth: what actually happened, established with certainty\nfrom any reliable source, including a fixing diff, tested source or later\ninvestigation. Production\'s answer is not an oracle. If you cannot establish\ntruth with certainty, the case is not ready: report it and keep collecting.\n\nBefore collecting, write the correct answer and the facts that establish it.\nTrace the original agent\'s tool calls, and the tools a successful agent would\nneed. Inventory the full retained environment: this trace, earlier runs and\ncomparisons, history up to the failure, source at the tested commit, config,\nlogs and media. Package it at full fidelity with time provenance. Hindsight is\nexcluded by time, not by deleting categories. Later fixes and investigations\nmay corroborate the label but never enter the agent\'s input.\n\nMissing evidence stays missing; label what a good agent should conclude given\nit. Never recreate missing DOM, logs or state and present them as historical\nevidence. Record the absence and the specific check that would decide it.\n\nKeep an evidence-map in case metadata, alongside the withheld expected value:\n\n  "evidence-map": {\n    "certainty": "Truth verified by fixing diff #123 and source at the tested commit.",\n    "facts": [\n      {"status":"proven", "fact":"The request failed", "path":"logs/network.jsonl",\n       "proof":"The request returned the recorded error before the cutoff."},\n      {"status":"absent", "fact":"The exact rendered state", "missing":"No DOM retained",\n       "expectedConclusion":"Name the missing DOM and the check it would settle."}\n    ]\n  }\n\nEvery label fact must be proven by a bundled path or explicitly absent with\nwhat a good agent should say. The certainty note can cite later investigation;\nthe map and expected answer stay outside the agent input. Self-check every\npath and explanation, and verify that no retained artifact is newer than the\nfailure cutoff. Do not use filesystem download times as evidence timestamps.\n\nGive full credit for reaching verified truth, or for identifying the specific\nmissing evidence that would decide it without asserting a contradictory cause.\nRequire the correct category where retained evidence supports it. Configure\nthat distinction in your grader; a vague request for more logs earns no gap\ncredit. `eval push` warns for stored cases without an evidence-map.\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval get <run-id> --case <key> --trace\n                                             model/tool timeline, errors and body links\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nModel request and response bodies are captured by default for new runs. Add\n`traces: { capture: metadata }` to a template to retain usage and capture state\nwithout storing bodies. `--trace` reads one case\'s trace a page at a time;\nlarge body content may be marked omitted while its individual body link remains\navailable. It also shows partial/capture errors and non-success model HTTP\nstatuses. `--json` returns the structured timeline and body references.\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. With the defaults (one\nrepeat, one to pass) a case passes when its single attempt passes. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Evidence map                 verified certainty; each label fact proven by a path or explicitly absent\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n\n\nTOOL EXECUTION AND EXTRA TRACES\n\nFull capture also keeps separate stdout/stderr prefixes (1 MiB each), the actual\nlast 16 KiB of combined output, and regular files under $EVAL_OUTPUT/traces/.\nWrite additional execution details there, for example:\n\n  mkdir -p "$EVAL_OUTPUT/traces"\n  printf \'%s\\n\' \'{"event":"tool_started","tool":"read_file"}\' >> "$EVAL_OUTPUT/traces/tools.jsonl"\n\nFiles are capped at 1 MiB each. Symlinks, hard links and non-regular files are\nrejected. One collection visits at most 128 entries; an execution accepts at\nmost 512 evidence artifacts plus a single count-limit marker. All artifacts\nshare the 64 MiB raw budget with model bodies. Truncation, interrupted streams,\nrejected files and storage failures remain visible; capture never turns a\npassing grade into a failure.\n\nAgents already configured for OpenTelemetry OTLP/HTTP tracing can use the\ninjected environment without adding export code:\n\n  OTEL_EXPORTER_OTLP_ENDPOINT          storage-only receiver base\n  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT   exact /otlp/v1/traces endpoint\n  OTEL_EXPORTER_OTLP_PROTOCOL          http/protobuf\n  OTEL_EXPORTER_OTLP_TRACES_PROTOCOL   http/protobuf\n  OTEL_EXPORTER_OTLP_HEADERS           scoped Authorization header\n  OTEL_EXPORTER_OTLP_TRACES_HEADERS    same scoped header\n\nThe receiver accepts OTLP JSON or protobuf, with optional gzip; each batch is\nlimited to 1 MiB compressed and 1 MiB raw, and intake is capped at 120 requests\nper minute. Tokens bind the org, attempt and execution generation. Capture\ncloses when the agent exits, so flush the agent\'s OTel provider before exiting.\nA hard-killed exporter can lose unflushed spans. No data is forwarded to a\nvendor, collector or monitoring service.\n\nIn metadata mode no endpoint or token is injected, process log bodies/tails\nand trace files are not retained, and OTLP payloads are rejected before parsing.\nOnly aggregate authenticated OTLP receipt counts are stored.\n\nREAD THE EVIDENCE\n\n  cofactor eval get <run-id> --case <key> --trace\n  cofactor eval get <run-id> --case <key> --trace --json\n\nThe output includes the model/tool timeline, failed tools, stdout/stderr,\nextra files and OTLP batch references with span counts. Body references are\nauthenticated API paths returning short-lived downloads of gzip objects.\nEarlier execution generations remain accessible through the API paths shown\nby the CLI.\n\nAfter the agent exits, script graders also receive EVAL_TRACE: a local JSON\nmanifest with a bounded timeline, capture errors, truncation status, local\nlog/file locations and API references. Its timeline reads at most 12 MiB raw\nand retains at most 768 KiB of entries. Follow API references when investigating\nan omitted body. This evidence is not automatically added to AI judge prompts;\nthe expected label still becomes available only during grading.\n\nCofactor\'s diagnostic eval writes tool-start/result/error events and its\nreasoning trail to these files. A start without an end identifies a stalled\nor interrupted tool. Reconstructed reasoning is labeled as reconstructed.\n';
+var overview_default6 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 1                      # attempts per case, 1 to 7 (default 1; for a stochastic agent raise it with passAtLeast)\n  passAtLeast: 1                  # a case passes when this many attempts pass (default 1, at most repeats)\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n  EVAL_TRACE_CAPTURE   full (default) or metadata\n  EVAL_NODE            the runner\'s trusted Node binary\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n[cli >= 0.56.0]\n- A wrong label or input is corrected with a new version, not by deleting the\n  case: `cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n  --expected <file|-> --source "<provenance>" --replace --reason "<why>" --org <org>`\n  stores version N+1 with who, when and the reason. Without `--replace` an\n  existing key is still an error. Runs that already used the case keep the\n  version they graded and show it as `key (vN)`; new runs use the latest.\n  `cofactor eval compare` refuses runs that graded different case versions, so\n  re-run the baseline after a correction.\n- `cofactor eval case <name> <key> [--version N]` prints a case\'s expected\n  label, provenance and version history (newest first).\n[/cli]\n\nCASE FIDELITY AND EVIDENCE MAP (REQUIRED)\n\nKeep three things separate:\n\n- The label is verified truth: what actually happened, established with\n  certainty by reliable evidence such as tested source, a verified fixing diff\n  or a later investigation. Production\'s answer is not an oracle. Neither is\n  the agent\'s own answer. If truth is still uncertain, report that the case is\n  not ready; do not turn a guess into a label.\n- The bundle is the environment as actually retained at failure time. It may\n  be incomplete even when the label is certain. Later findings may establish\n  the label but must not be supplied to the agent as failure-time evidence.\n- Missing evidence stays missing. Never reproduce missing DOM, logs, video or\n  state for the case. A later reproduction is not the original evidence. A\n  case can test whether the agent identifies exactly what it cannot know.\n\nBUILD THE BUNDLE\n\n1. Write the verified answer and how you verified it in a separate expected\n   JSON file. List the facts needed to reach that answer. Keep this file out\n   of the input directory and out of the repository used by the agent.\n2. Set the failure cutoff and record the tested commit. Inventory what was\n   actually retained: the original tool calls and results, earlier runs and\n   comparisons, history up to the cutoff, tested-commit source, configuration,\n   logs, screenshots and recordings. Include what a successful agent\'s tools\n   would need, not just the files the failing agent happened to read.\n3. Collect the full retained artifacts, including earlier-run context. Keep\n   timestamps and source/version provenance. Exclude later outcomes, fixes and\n   investigation notes from input. Use event/capture times, not filesystem\n   download times. History and archives are valid evidence when cutoff-bounded.\n4. Distinguish an original absence from a collection failure. If a retained\n   recording cannot be downloaded, fix collection before uploading the case;\n   do not describe it as never retained. Record genuinely absent artifacts,\n   partial capture and truncation as such. Never fill the gaps by rerunning.\n5. Sanitize consistently across text, source, images, video and archives.\n   Replace names and domains with Acme and example.com; remove credentials and\n   identifying data. Preserve timing, relationships and behavior. Review\n   images and binaries as well as text. If removing sensitive material loses\n   a decisive fact, mark it absent and adjust the expected conclusion, or hold\n   the case back. Do not silently replace evidence with a summary.\n6. Record each artifact\'s relative path, origin, capture time, byte count,\n   hash and availability in the bundle inventory. Choose a byte budget before\n   packaging; retain full files rather than excerpts. If the environment will\n   not fit, report the limit rather than silently dropping evidence. Document\n   any explicitly dispensable removal and why it cannot affect the answer.\n7. Build the evidence map below. Open every proven path, check its contents\n   against the claimed proof, and verify the cutoff. Confirm that expected,\n   the map and later corroboration are outside the input and unreachable by\n   the agent\'s tools. Point historical tools at captured files, not live data.\n\nBUILD THE EVIDENCE MAP\n\nStore the map inside the withheld expected JSON as expected["evidence-map"].\nIt is grading metadata, never an agent input or an extra YAML template field.\nThe input inventory describes what is available; the map explains what that\nevidence proves about the label.\n\n- certainty: how truth was independently verified; may cite later findings.\n- facts: one entry for every fact the label relies on, with one of two states:\n  - proven: fact, path relative to the input directory, and proof explaining\n    exactly what in that file establishes the fact.\n  - absent: fact, missing naming the specific evidence unavailable in input,\n    and expectedConclusion saying what the agent should conclude or check.\n\nAn absent fact means the bundle cannot establish it, not that it is false.\nDo not mark a cause proven just because the label\'s author knows it. Map all\nfacts needed for the cause, not only the visible symptom. `eval push` warns\nfor stored cases without an evidence-map; that warning does not validate a\nmap\'s paths, completeness or truth. Review them yourself.\n\nWORKED EXAMPLE: VERIFIED LABEL\n\nThese are invented, neutral teaching cases. Mark them synthetic in --source;\ndo not describe them as captured production incidents. Paths and expected\nfields are examples for your own entrypoint and graders, not a required schema\nbeyond the evidence-map shape.\n\nAcme\'s checkout at https://example.com/checkout failed. The production agent\nsaid the session had expired. Independently checked source and retained config\nshow that checkout was disabled, and the response names that reason. Label the\ndisabled checkout, even though it disagrees with production\'s diagnosis.\n\nKeep the full retained files in a layout such as:\n\n  verified-label/\n    input/\n      input.json                  request, failure cutoff and tested commit\n      config/checkout.json        {"checkoutEnabled":false}\n      logs/network.jsonl          response: 403, reason: checkout_disabled\n      source/checkout.ts          tested source: disabled checkout returns that reason\n      traces/tools.jsonl          original tool calls and results\n      history/                    earlier runs/comparisons, if retained\n      media/                      original images/recordings, if retained\n      inventory.json              artifact provenance, sizes, hashes and absences\n    expected.json                 withheld; never inside input/\n\nThe short file descriptions above explain the example; package the full files.\nFor this teaching case, expected.json contains:\n\n  {\n    "rootCause": "Checkout was disabled in Acme\'s configuration.",\n    "evidence-map": {\n      "certainty": "Independently checked the retained config, response and tested source; production\'s expired-session diagnosis was wrong.",\n      "facts": [\n        {"status":"proven", "fact":"Checkout was disabled",\n         "path":"config/checkout.json", "proof":"checkoutEnabled is false."},\n        {"status":"proven", "fact":"The request was rejected because checkout was disabled",\n         "path":"logs/network.jsonl", "proof":"The checkout response is 403 with reason checkout_disabled."},\n        {"status":"proven", "fact":"That response reason identifies the disabled-checkout branch",\n         "path":"source/checkout.ts", "proof":"At the tested commit, that reason is returned only when checkoutEnabled is false."}\n      ]\n    }\n  }\n\nA passing answer cites the config and response and explains the disabled\ncheckout. Repeating the production diagnosis fails. Judge the evidence and\nmeaning, not whether the answer copies the label\'s words.\n\n  cofactor eval add-case acme-diagnostics --key checkout-disabled \\\n    --input ./verified-label/input --expected ./verified-label/expected.json \\\n    --source "Synthetic teaching case: independently checked config, response and tested source" --org <org>\n\nWORKED EXAMPLE: MISSING EVIDENCE\n\nAcme\'s Save click at https://example.com/settings timed out. The retained tool\ntrace records a timeout, but no DOM snapshot or recording from the failed click\nwas retained. In this invented case, a later incident investigation and verified\nfix established that a loading overlay intercepted the click. That establishes\nthe label; it does not give the agent the missing failure-time evidence.\n\nBuild missing-evidence/input with input.json, the complete traces/tools.jsonl,\nthe retained tested source/config/history and an inventory of what is present\nor absent. Do not create a replacement DOM or recording. Keep the later\ninvestigation and fix outside input. In missing-evidence/expected.json, write:\n\n  {\n    "rootCause": "A loading overlay intercepted the Save click.",\n    "evidence-map": {\n      "certainty": "The later incident investigation and verified fix established the overlay interception; neither is in the agent input.",\n      "facts": [\n        {"status":"proven", "fact":"The Save click timed out",\n         "path":"traces/tools.jsonl", "proof":"The Save click tool result records a timeout."},\n        {"status":"absent", "fact":"An overlay intercepted the failed click",\n         "missing":"No failure-time DOM with hit-target information or recording of the Save click was retained.",\n         "expectedConclusion":"Name the missing failure-time DOM/hit-target evidence or recording. Explain that inspecting what covered Save at the click would distinguish an intercepting overlay from a disabled button or a stale target. Keep the cause unresolved without it."}\n      ]\n    }\n  }\n\nA passing answer says, for example: "The Save click timed out. There is no DOM\nwith hit-target information or recording from that click, so I cannot tell\nwhether an overlay covered Save, the button was disabled or the target was\nstale. That evidence would show which element received or blocked the click."\nIt names a check; it does not rerun the page to manufacture the missing past.\n\n"Need more logs" is too vague. "An overlay definitely blocked Save" also fails:\nit guesses the withheld truth without support. A tentative hypothesis must\nremain tentative and still name the decisive missing evidence and check.\n\n  cofactor eval add-case acme-diagnostics --key save-evidence-missing \\\n    --input ./missing-evidence/input --expected ./missing-evidence/expected.json \\\n    --source "Synthetic teaching case: truth verified later; failure-time DOM and recording absent" --org <org>\n\nWRITE THE JUDGE\'S RULE\n\nConfigure your grader to use the evidence map. Give full credit for a verified\ncause supported by retained evidence, or for naming the specific decisive\nmissing evidence and the check that distinguishes competing causes. Do not\nreward a lucky guess at withheld truth. Unrelated absences do not disqualify\nan otherwise proven cause. Require the correct category when retained evidence\nsupports it; do not force a category the agent could not determine.\n\nThe judge\'s own explanation must name the specific missing evidence and the\ncheck, not just say "insufficient evidence". A rubric for these examples can say:\n\n  Pass when the answer either explains the verified cause using retained\n  evidence, or identifies the decisive missing evidence from the evidence map\n  and explains how checking it would distinguish the competing causes. Fail\n  generic requests for more data and unsupported certain causes, even if a\n  guess matches the label. In your grading reason, name the particular missing\n  evidence and distinguishing check whenever absence affects the grade.\n\nFor a 1-5 diagnostic judge with passAt 4, vague or incomplete gaps and\nunsupported certain causes score at most 3 when decisive evidence is absent.\nCheck the rubric against the passing and failing answers above before using it.\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval get <run-id> --case <key> --trace\n                                             model/tool timeline, errors and body links\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nModel request and response bodies are captured by default for new runs. Add\n`traces: { capture: metadata }` to a template to retain usage and capture state\nwithout storing bodies. `--trace` reads one case\'s trace a page at a time;\nlarge body content may be marked omitted while its individual body link remains\navailable. It also shows partial/capture errors and non-success model HTTP\nstatuses. `--json` returns the structured timeline and body references.\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. With the defaults (one\nrepeat, one to pass) a case passes when its single attempt passes. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Evidence map                 verified certainty; each label fact proven by a path or explicitly absent\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n\n\nTOOL EXECUTION AND EXTRA TRACES\n\nFull capture also keeps separate stdout/stderr prefixes (1 MiB each), the actual\nlast 16 KiB of combined output, and regular files under $EVAL_OUTPUT/traces/.\nWrite additional execution details there, for example:\n\n  mkdir -p "$EVAL_OUTPUT/traces"\n  printf \'%s\\n\' \'{"event":"tool_started","tool":"read_file"}\' >> "$EVAL_OUTPUT/traces/tools.jsonl"\n\nFiles are capped at 1 MiB each. Symlinks, hard links and non-regular files are\nrejected. One collection visits at most 128 entries; an execution accepts at\nmost 512 evidence artifacts plus a single count-limit marker. All artifacts\nshare the 64 MiB raw budget with model bodies. Truncation, interrupted streams,\nrejected files and storage failures remain visible; capture never turns a\npassing grade into a failure.\n\nAgents already configured for OpenTelemetry OTLP/HTTP tracing can use the\ninjected environment without adding export code:\n\n  OTEL_EXPORTER_OTLP_ENDPOINT          storage-only receiver base\n  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT   exact /otlp/v1/traces endpoint\n  OTEL_EXPORTER_OTLP_PROTOCOL          http/protobuf\n  OTEL_EXPORTER_OTLP_TRACES_PROTOCOL   http/protobuf\n  OTEL_EXPORTER_OTLP_HEADERS           scoped Authorization header\n  OTEL_EXPORTER_OTLP_TRACES_HEADERS    same scoped header\n\nThe receiver accepts OTLP JSON or protobuf, with optional gzip; each batch is\nlimited to 1 MiB compressed and 1 MiB raw, and intake is capped at 120 requests\nper minute. Tokens bind the org, attempt and execution generation. Capture\ncloses when the agent exits, so flush the agent\'s OTel provider before exiting.\nA hard-killed exporter can lose unflushed spans. No data is forwarded to a\nvendor, collector or monitoring service.\n\nIn metadata mode no endpoint or token is injected, process log bodies/tails\nand trace files are not retained, and OTLP payloads are rejected before parsing.\nOnly aggregate authenticated OTLP receipt counts are stored.\n\nREAD THE EVIDENCE\n\n  cofactor eval get <run-id> --case <key> --trace\n  cofactor eval get <run-id> --case <key> --trace --json\n\nThe output includes the model/tool timeline, failed tools, stdout/stderr,\nextra files and OTLP batch references with span counts. Body references are\nauthenticated API paths returning short-lived downloads of gzip objects.\nEarlier execution generations remain accessible through the API paths shown\nby the CLI.\n\nAfter the agent exits, script graders also receive EVAL_TRACE: a local JSON\nmanifest with a bounded timeline, capture errors, truncation status, local\nlog/file locations and API references. Its timeline reads at most 12 MiB raw\nand retains at most 768 KiB of entries. Follow API references when investigating\nan omitted body. This evidence is not automatically added to AI judge prompts;\nthe expected label still becomes available only during grading.\n\nCofactor\'s diagnostic eval writes tool-start/result/error events and its\nreasoning trail to these files. A start without an end identifies a stalled\nor interrupted tool. Reconstructed reasoning is labeled as reconstructed.\n';
 
 // ../content/src/eval-guide/index.ts
 var EVAL_GUIDE_SECTIONS = [
@@ -2186,4 +2186,4 @@ export {
   shouldBrandConsoleOutput,
   installConsoleBranding
 };
-//# sourceMappingURL=chunk-T6ZNIMI4.js.map
+//# sourceMappingURL=chunk-LRIP5IQD.js.map