@runcofactor/cli 0.55.0 → 0.56.0

package/dist/{account-RHHS7HT4.js → account-CZYZD2JS.js}

@@ -3,17 +3,17 @@ import {
   apiRequest,
   fetchList,
   getTargetOrg
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/account.ts
@@ -254,4 +254,4 @@ async function runAccount(argv) {
 export {
   runAccount
 };
-//# sourceMappingURL=account-RHHS7HT4.js.map
+//# sourceMappingURL=account-CZYZD2JS.js.map

package/dist/{alert-B5WEOKHP.js → alert-JQ4SDEB7.js}

@@ -20,25 +20,25 @@ import {
   mergeSteps,
   parseFilters,
   updateRule
-} from "./chunk-YTXZUWMN.js";
-import "./chunk-XM3WTJ7Y.js";
-import "./chunk-XWJJYZQC.js";
+} from "./chunk-62EXS76C.js";
+import "./chunk-DEVD2WD2.js";
+import "./chunk-XHEVDABB.js";
 import "./chunk-DEK6B6LX.js";
 import {
   fetchList,
   getPositionalArg
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   GLOBAL_VALUE_FLAGS,
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/alert.ts
@@ -315,4 +315,4 @@ async function runAlert(argv) {
 export {
   runAlert
 };
-//# sourceMappingURL=alert-B5WEOKHP.js.map
+//# sourceMappingURL=alert-JQ4SDEB7.js.map

package/dist/{api-catalog-UXONX3BR.js → api-catalog-IWQC35P2.js}

@@ -1,16 +1,16 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
 import {
   apiRequest
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   getArgValue,
   hasFlag
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/api-catalog.ts
@@ -414,4 +414,4 @@ function exitErr(msg) {
 export {
   runApiCatalog
 };
-//# sourceMappingURL=api-catalog-UXONX3BR.js.map
+//# sourceMappingURL=api-catalog-IWQC35P2.js.map

package/dist/{api-key-J3A4TJKN.js → api-key-GUDXIIEA.js}

@@ -2,17 +2,17 @@ import { createRequire as __cr } from "module"; import { fileURLToPath as __fup
 import {
   apiRequest,
   getArgValues
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/api-key.ts
@@ -176,4 +176,4 @@ export {
   createApiKey,
   runApiKey
 };
-//# sourceMappingURL=api-key-J3A4TJKN.js.map
+//# sourceMappingURL=api-key-GUDXIIEA.js.map

package/dist/{attachment-ZQW7DR4G.js → attachment-CPZYOBUB.js}

@@ -1,17 +1,17 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
 import {
   apiRequest
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/attachment.ts
@@ -275,4 +275,4 @@ async function runAttachment(argv) {
 export {
   runAttachment
 };
-//# sourceMappingURL=attachment-ZQW7DR4G.js.map
+//# sourceMappingURL=attachment-CPZYOBUB.js.map

package/dist/{auth-HQGKUZBS.js → auth-DS6MBVEI.js}

@@ -27,8 +27,8 @@ import {
   resolveOrgAppUrl,
   saveAuth,
   setDefaultProfile
-} from "./chunk-BJHAVM7Z.js";
-import "./chunk-KL2ARGH6.js";
+} from "./chunk-4MFMPC5P.js";
+import "./chunk-2OHYMT3A.js";
 import "./chunk-TOWZAG3I.js";
 export {
   ENV_URLS,
@@ -59,4 +59,4 @@ export {
   saveAuth,
   setDefaultProfile
 };
-//# sourceMappingURL=auth-HQGKUZBS.js.map
+//# sourceMappingURL=auth-DS6MBVEI.js.map

package/dist/{benchmark-TIKJA32J.js → benchmark-6HPCIQHD.js}

@@ -3,18 +3,18 @@ import {
   apiRequest,
   fetchList,
   getPositionalArgs
-} from "./chunk-WWUEKQLR.js";
+} from "./chunk-EWK44F6V.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   GLOBAL_VALUE_FLAGS,
   getArgValue,
   hasFlag,
   resolveConfig
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
-import "./chunk-U3MRXQZS.js";
+} from "./chunk-2OHYMT3A.js";
+import "./chunk-RHWPZKRV.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/benchmark.ts
@@ -437,4 +437,4 @@ export {
   parseAbArgs,
   runBenchmark
 };
-//# sourceMappingURL=benchmark-TIKJA32J.js.map
+//# sourceMappingURL=benchmark-6HPCIQHD.js.map

package/dist/{blackcat-Z5LNV2GS.js → blackcat-V3L63WFE.js}

@@ -7,10 +7,10 @@ import {
   isTerminalFailure,
   resolveConfig,
   splitAtSeparator
-} from "./chunk-BJHAVM7Z.js";
+} from "./chunk-4MFMPC5P.js";
 import {
   writeRawOutput
-} from "./chunk-KL2ARGH6.js";
+} from "./chunk-2OHYMT3A.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/blackcat.ts
@@ -268,4 +268,4 @@ export {
   runBlackcat,
   waitForBlackcatResume
 };
-//# sourceMappingURL=blackcat-Z5LNV2GS.js.map
+//# sourceMappingURL=blackcat-V3L63WFE.js.map

package/dist/{chunk-JQG3V5E3.js → chunk-2E3STBHS.js}

@@ -2,7 +2,7 @@ import { createRequire as __cr } from "module"; import { fileURLToPath as __fup
 import {
   redactCredentialBody,
   redactJsonInPlace
-} from "./chunk-U3MRXQZS.js";
+} from "./chunk-RHWPZKRV.js";
 import {
   __commonJS,
   __require,
@@ -3735,4 +3735,4 @@ export {
   extractScreencastFramesToDir,
   inProcessTraceReader
 };
-//# sourceMappingURL=chunk-JQG3V5E3.js.map
+//# sourceMappingURL=chunk-2E3STBHS.js.map

package/dist/{chunk-KL2ARGH6.js → chunk-2OHYMT3A.js}

@@ -1685,7 +1685,7 @@ var ISSUES_GUIDE_SECTIONS = [
 ];
 
 // ../content/src/proofs-guide/overview.txt
-var overview_default4 = 'PROVE A CHANGE WITH TESTS\n\nUse this guide for a pull request that changes user-visible behavior. The pull\nrequest connects numbered acceptance criteria to passing Workflow evidence.\nPass `--org <org>` on every write to select the organization.\n\nCHOOSING PROOFS\n\nPick the proof that fits the change; one change can need several.\n\n- Functional change: a Workflow run that passes at the pushed commit. The\n  procedure below covers it.\n- Performance-sensitive change: a perf run. Read `cofactor perf guide`. A\n  query change, for example, needs both a functional proof and a perf proof.\n- UI change: a Workflow with `visual_checkpoint` nodes. Read\n  `cofactor workflow guide visual-checkpoints`, and step 7 below for how\n  attaching runs the test at the PR base and shows the checkpoint diffs.\n  When the organization uses visual regression (its Workflows already carry\n  `visual_checkpoint` nodes, or its admin says Visual checkpoints are on), a\n  change that alters what a screen looks like must carry UI-test evidence: run a Workflow with a `visual_checkpoint` on that screen, adding or\n  extending one if none exists. A manual screenshot does not replace it. With\n  visual regression off, skip this and say so in the criterion\'s `--detail`.\n[cli >= 0.54.0]\n- Agent behavior: evals. Read `cofactor eval guide`, then attach a full passing\n  eval run with `--eval-run` (see step 6).\n[/cli]\n\nTHE PROCEDURE\n\n1. Find coverage before writing a Workflow:\n     cofactor workflow list --search "<words>"\n     cofactor workflow get <wf_...> --format markdown\n   Read the candidate assertions. If a test already walks the journey, use it.\n   Never add near-duplicates or a new test for a changed filter, column or field.\n\n2. Choose the Workflow:\n   - If an approved Workflow already proves the criterion without changes (the\n     journey must still work, for example after a query or refactor change),\n     run it as is and attach its run. It is cited as an existing test: it\n     proves the journey still works, it is not new coverage, and it is never\n     promoted (it is already in its suites). It cannot prove new behavior: a\n     new filter, column or field needs a fork that asserts it. An approved\n     fork is still treated as a fork; to cite an existing test, attach the\n     approved test that is not a fork.\n   - Fork an approved Workflow with\n     `cofactor workflow fork fork <wf_...> --org <org>` and edit the fork.\n     Never edit the approved original.\n   - Create a new draft only when no existing journey fits or it needs its own\n     data or role (see `cofactor workflow guide write-a-test`).\n\n3. Configure promotion on the fork or new draft before attaching evidence:\n     cofactor workflow set-promotion <workflowId> --promote-on merge|never|deploy:<stage> [--target-suite <suite>]... --org <org>\n   Repeat `--target-suite` for multiple suites. The command replaces the target\n   suite list; omit it to clear extra suites. Promotion adds target suites and\n   the default regression suite when present. An org without one can use the\n   **Create default suite** button. Do not choose a layer.\n   `never` archives the test at merge instead of adding it to any suite, and\n   takes no `--target-suite`; use it when the proof is about the change itself\n   (see CURATING THE SUITE). It has no effect on an already-approved test.\n\n4. Prove the change at the exact pushed commit. Start from a clean checkout,\n   commit and push the application change, then use the pushed commit from\n   `git rev-parse HEAD` as `<sha>` in `--at <sha>`. Claim that SHA and sync it:\n     cofactor sandbox claim --application <app> --at <sha> --wait --org <org>\n     cofactor sandbox sync <sandboxId> [--build] --org <org>\n     cofactor workflow run <workflowId> --sandbox <sandboxId> --stream --org <org>\n   Keep the sandbox id. If you lose it, `cofactor sandbox claims --org <org>`\n   lists your live claims, and `cofactor sandbox destroy <sandboxId>` tears one\n   down when you are done. The run uses the claimed sandbox\'s own environment,\n   so do not pass `--environment`.\n   Iterate until it passes, then run it once more after the last application or\n   Workflow test change and use that passing run ID. An existing test is run the\n   same way, at the pushed commit and on its current revision. For application edits,\n   commit and push, then sync and rerun. Leave the checkout clean. Attach\n   refuses dirty checkouts or missing commit evidence. If the run records a\n   synced tree, it must equal `HEAD^{tree}`; otherwise its recorded source SHA\n   must equal `HEAD`. The run must exist, belong to the cited Workflow, pass,\n   and use its current revision. Attach requires a GitHub-delivered application;\n   upload-delivered applications cannot supply this proof.\n\n5. After proving, commit any remaining changes and push the branch. If those\n   commits change the application or Workflow tests, return to step 4 and\n   prove them before proceeding. Open a DRAFT pull request and note its number\n   `<n>`; attach needs that PR number.\n\n6. Write concise numbered acceptance criteria. State observable outcomes\n   plainly. One run may support several criteria; multiple runs may support one\n   criterion. For each evidence item, use the most specific node or assertion\n   ID available and state what it proves as `Asserted that \u2026`. Omit optional\n   `--node` only when there is no specific ID.\n   A performance run can prove a criterion too, and one criterion can mix\n   kinds: for a query change, attach a Workflow run for the behavior and a\n   performance run for the speed on the same criterion number. Run it at the pushed head SHA\n   and attach it with `--perf-run <prun_...>`; one criterion can carry several\n   runs, and every attached item must pass. A performance run must be completed,\n   compare different base and head commits, and be at the commit you attach; a\n   slower result is refused. Attach a new run at the current head, then detach\n   an older stale item when the criterion should become proved again.\n   [cli >= 0.54.0]\n   An agent eval run can prove a criterion as well (a change to an agent\'s\n   behavior): attach it with `--eval-run <aerun_...>`. It must be a full run\n   (every case at its repeats), completed, passing its pass bar, and measured at\n   the commit you attach; a partial, failing or other-commit run is refused.\n   One criterion can carry any number of eval runs, each with its own model, and\n   every one must pass.\n   [/cli]\n\n7. After the draft PR exists, attach evidence. The server refuses a new\n   criterion number without both `--text` and `--detail`; reuse the same text\n   and detail for additional evidence on that number:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --test <wf_...> --run <wrun_...> [--node <most-specific-id>] [--base <sha>] --asserted "Asserted that \u2026" --org <org>\n   For performance evidence, use `--perf-run` instead of `--test` and `--run`:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --perf-run <prun_...> --asserted "Asserted that \u2026" --org <org>\n   [cli >= 0.54.0]\n   For an eval run, use `--eval-run` in the same place:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_...> --asserted "Asserted that \u2026" --org <org>\n   [/cli]\n   An approved Workflow that is not a fork is attached the same way; the\n   result says `"existing": true`.\n   Attach exactly one evidence subject per command. A test containing any\n   `visual_checkpoint` node is a UI test, including a test that also has other\n   steps. Attaching it runs the same test at the PR base commit and the proof\n   page shows each checkpoint\'s base, head, and diff. When `--base` is omitted,\n   the CLI uses `git merge-base HEAD origin/<default>` when that ref is available.\n   Put checkpoints before\n   steps that depend on the new behavior so the base run can reach them.\n\n8. Push first, then render the criteria block:\n     cofactor proofs render --pr <n> --org <org>\n   The output already includes the markers. Paste the entire output into the\n   PR description, verbatim, replacing any existing proof block. Never\n   hand-format or edit it, and do not add a prose "Proof notes" section that\n   restates it. It looks like this:\n\n     <!-- cofactor:proof -->\n\n     **Proof of acceptance criteria** \xB7 [View all proofs in Cofactor](<proof page>)\n\n     \u{1F7E2} 1 passed \xB7 \u{1F7E1} 1 stale \xB7 \u26AA 1 manual \xB7 as of `a1b2c3d`\n\n     \u{1F7E2} **1. The saved filter stays selected after a reload** \xB7 Passed\n     - **Tested:** Filter bar keeps its selection \xB7 E2E run in a sandbox \xB7 [View the run](<run page>)\n       - Asserted that the Status filter still shows "Open" after a reload.\n\n     \u{1F7E1} **2. The list query is no slower** \xB7 Stale (proved at 9f8e7d6)\n     - **Tested:** Performance run (prod) \xB7 [See the comparison](<perf run page>)\n       - Asserted that the list query is no slower than base.\n\n     \u26AA **3. The desktop client shows the badge** \xB7 Manual\n     - **Tested:** Manual check, not provable with Cofactor \xB7 [See the proof](<proof page>)\n       - Opened the desktop build and checked the badge.\n\n   Each criterion leads with a stoplight: \u{1F7E2} passed, \u{1F7E1} passed at an earlier\n   commit (stale), \u{1F534} failed, \u26AA shown manually because Cofactor could not prove\n   it. Under it, each evidence item names the test (its user goal), how it\n   ran, what it asserted, and links to it. The block is a snapshot as of the\n   commit in its header: it goes stale after a push, and the live verdict is\n   the `cofactor / proofs` check and the proof page. After every push, attach,\n   or detach, render again and replace the block. Context for a reviewer goes\n   in the criterion\'s `--detail` and in the `Asserted that \u2026` line (the proof\n   page shows both), not in a paragraph in the PR body.\n\n9. Inspect or revise the proof:\n     cofactor proofs status --repo <owner/repo> --pr <n> --org <org>\n   The repo is inferred from `GITHUB_REPOSITORY` or `git remote get-url origin`\n   and can be overridden with `--repo`.\n   Detaching removes a criterion and all evidence attached to it:\n     cofactor proofs detach --pr <n> --criterion <n> --org <org>\n   Remove just one item with `--test <wf_...>`, `--perf-run <prun_...>` or\n   `--manual`:\n     cofactor proofs detach --pr <n> --criterion <n> --perf-run <prun_...> --org <org>\n   [cli >= 0.54.0]\n   `--eval-run <aerun_...>` removes one eval run the same way.\n   [/cli]\n   Render again and replace the PR proof block after every attach or detach.\n\nWHEN COFACTOR CANNOT PROVE A CRITERION\n\nManual evidence is for a criterion Cofactor cannot exercise at all (for example\nit needs a desktop client, real hardware or a third-party console). If the\ncriterion is plainly outside a browser or a sandbox, go straight to it;\notherwise try a Cofactor test first. A failing, flaky or hard-to-write test is\nnever a reason for manual evidence: fix the test or the change, and never detach\nfailing evidence to replace it with manual evidence.\n\nSay what you did and why Cofactor could not prove it, from a clean checkout at\nthe pushed commit:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --manual --notes <text|@file> [--artifact <https-url>]... --org <org>\n   Notes (up to 10,000 characters) and links (up to 20) are shown to everyone\n   who can read the proof: keep notes to what you did and why, and redact\n   secrets, tokens and customer data from anything you link. Links must be\n   https URLs reviewers can open, never signed or tokenized links and never\n   with credentials (an image attached to the pull request works; where your\n   organization has Cofactor utilities, `cofactor utilities object upload\n   <file> --public --json` returns one).\n   A criterion holds one manual item, and attaching again replaces its notes\n   and links, so pass the same `--notes` and every `--artifact` again. A\n   criterion whose only evidence is manual passes the check with `\u{1F4DD}` and\n   `[manual]`, renders \u26AA Manual in the block, and shows a Manual badge with\n   your notes on the proof page; a criterion that mixes manual with other kinds\n   keeps its stoplight and is decided by its other items. Manual evidence never promotes\n   anything and reads stale after a push; attach it again at the new head.\n\nCHECKS AND STALE EVIDENCE\n\nThe App publishes `cofactor / proofs`; the Actions job reports the same server\nverdict. Diagnose with a body file:\n     cofactor proofs check --pr <n> --sha <head> --body-file <file> --org <org>\n   Or pass `--body -` for stdin. The App queues a refreshed check after attach\nor detach. In the Actions path, rerun the check job after attaching or\ndetaching. Successful stale evidence remains successful and is\nmarked stale. Use Re-run on the Cofactor proof page for stale E2E evidence at\nthe PR head. For stale performance evidence, run it at the new head, attach\nthat run and detach the old item. Cofactor does not rerun automatically. Server and CI own promotion\ntriggers; `cofactor trigger` is not your job.\n\nNo block means no proof check; a block with no criteria fails. When the pull\nrequest has no observable acceptance criterion at all (for example a\ndependency bump or a docs-only change), print the neutral block and paste it\nin place of any proof block:\n     cofactor proofs render --none "<why this pull request has no criterion to prove>"\nIt reports a neutral result. A criterion that only Cofactor cannot exercise\ngets manual evidence instead (above).\n\nCURATING THE SUITE\n\nEvery promoted test runs from then on, so keep the suite worth running:\n   - Reuse first: attach the run of an existing test or eval that already proves\n     the criterion before creating one.\n   - Extend an existing test instead of duplicating its journey with a new one.\n   - One test per user goal, named for that goal (what the user accomplishes,\n     not the component or the ticket).\n   - Promote lasting behaviour whose regression would matter.\n   - Set a Workflow to `--promote-on never` when its proof is about the change\n     itself: a one-time migration, backfill, redirect or flag rollout step,\n     behaviour about to be removed, or a one-off visual before/after of a\n     redesign.\n   - Perf and eval runs never join a suite through proofs: a perf A/B or an\n     eval experiment for one decision stays a run. Save a perf test only for a\n     hot path (see the perf guide).\n   - Don\'t keep a test nobody would miss if it disappeared.\n\nWHAT GOES IN THE PULL REQUEST\n\nPaste the output of `proofs render` and nothing that restates it: no "Proof\nnotes" section. If a reviewer needs to know why a Workflow was forked or what\nstays uncovered, say it once in the criterion\'s `--detail`, or make it a\ncriterion with manual evidence.\n\nUse neutral example names and never include customer names, domains, or\ncredentials. A human approves Workflows by merging the pull request; do not\napprove them yourself.\n';
+var overview_default4 = 'PROVE A CHANGE WITH TESTS\n\nUse this guide for a pull request that changes user-visible behavior. The pull\nrequest connects numbered acceptance criteria to passing Workflow evidence.\nPass `--org <org>` on every write to select the organization.\n\nCHOOSING PROOFS\n\nPick the proof that fits the change; one change can need several.\n\n- Functional change: a Workflow run that passes at the pushed commit. The\n  procedure below covers it.\n- Performance-sensitive change: a perf run. Read `cofactor perf guide`. A\n  query change, for example, needs both a functional proof and a perf proof.\n- UI change: a Workflow with `visual_checkpoint` nodes. Read\n  `cofactor workflow guide visual-checkpoints`, and step 7 below for how\n  attaching runs the test at the PR base and shows the checkpoint diffs.\n  For sandbox-only tests, prepare visual data with the test\'s own SQL fixture\n  through the app\'s generic loader. That guide covers adding it and API/UI\n  setup for tests that must also run against staging or prod.\n  When the organization uses visual regression (its Workflows already carry\n  `visual_checkpoint` nodes, or its admin says Visual checkpoints are on), a\n  change that alters what a screen looks like must carry UI-test evidence: run a Workflow with a `visual_checkpoint` on that screen, adding or\n  extending one if none exists. A manual screenshot does not replace it. With\n  visual regression off, skip this and say so in the `Asserted that \u2026` line of another item,\n  or mark the criterion not proved with that reason (see below).\n[cli >= 0.54.0]\n- Agent behavior: evals. Read `cofactor eval guide`, then attach a full passing\n  eval run with `--eval-run` (see step 6).\n[/cli]\n\nTHE PROCEDURE\n\n1. Find coverage before writing a Workflow:\n     cofactor workflow list --search "<words>"\n     cofactor workflow get <wf_...> --format markdown\n   Read the candidate assertions. If a test already walks the journey, use it.\n   Never add near-duplicates or a new test for a changed filter, column or field.\n\n2. Choose the Workflow:\n   - If an approved Workflow already proves the criterion without changes (the\n     journey must still work, for example after a query or refactor change),\n     run it as is and attach its run. It is cited as an existing test: it\n     proves the journey still works, it is not new coverage, and it is never\n     promoted (it is already in its suites). It cannot prove new behavior: a\n     new filter, column or field needs a fork that asserts it. An approved\n     fork is still treated as a fork; to cite an existing test, attach the\n     approved test that is not a fork.\n   - Fork an approved Workflow with\n     `cofactor workflow fork fork <wf_...> --org <org>` and edit the fork.\n     Never edit the approved original.\n   - Create a new draft only when no existing journey fits or it needs its own\n     data or role (see `cofactor workflow guide write-a-test`).\n\n3. Configure promotion on the fork or new draft before attaching evidence:\n     cofactor workflow set-promotion <workflowId> --promote-on merge|never|deploy:<stage> [--target-suite <suite>]... --org <org>\n   Repeat `--target-suite` for multiple suites. The command replaces the target\n   suite list; omit it to clear extra suites. Promotion adds target suites and\n   the default regression suite when present. An org without one can use the\n   **Create default suite** button. Do not choose a layer.\n   `never` archives the test at merge instead of adding it to any suite, and\n   takes no `--target-suite`; use it when the proof is about the change itself\n   (see CURATING THE SUITE). It has no effect on an already-approved test.\n\n4. Prove the change at the exact pushed commit. Start from a clean checkout,\n   commit and push the application change, then use the pushed commit from\n   `git rev-parse HEAD` as `<sha>` in `--at <sha>`. Claim that SHA and sync it:\n     cofactor sandbox claim --application <app> --at <sha> --wait --org <org>\n     cofactor sandbox sync <sandboxId> [--build] --org <org>\n     cofactor workflow run <workflowId> --sandbox <sandboxId> --stream --org <org>\n   Keep the sandbox id. If you lose it, `cofactor sandbox claims --org <org>`\n   lists your live claims, and `cofactor sandbox destroy <sandboxId>` tears one\n   down when you are done. The run uses the claimed sandbox\'s own environment,\n   so do not pass `--environment`.\n   Iterate until it passes, then run it once more after the last application or\n   Workflow test change and use that passing run ID. An existing test is run the\n   same way, at the pushed commit and on its current revision. For application edits,\n   commit and push, then sync and rerun. Leave the checkout clean. Attach\n   refuses dirty checkouts or missing commit evidence. If the run records a\n   synced tree, it must equal `HEAD^{tree}`; otherwise its recorded source SHA\n   must equal `HEAD`. The run must exist, belong to the cited Workflow, pass,\n   and use its current revision. Attach requires a GitHub-delivered application;\n   upload-delivered applications cannot supply this proof.\n\n5. After proving, commit any remaining changes and push the branch. If those\n   commits change the application or Workflow tests, return to step 4 and\n   prove them before proceeding. Open a DRAFT pull request and note its number\n   `<n>`; attach needs that PR number.\n\n6. Write concise numbered acceptance criteria. A criterion is one short\n   observable outcome a user can see, for example "An external run started with\n   triage off stays off". Write the criterion, `--text` and `--asserted` in\n   product language a reviewer understands: what the user sees or what the\n   product does. Leave out field names, column names, flags, IDs, UUIDs, SHAs\n   and code identifiers; the reviewer reads them in the PR and cannot interpret\n   them. For example:\n     Bad:  Asserted that an API-launched external run reads back triageEnabled=false\n     Good: Asserted that an external run launched with triage turned off stays\n           off, and defaults to on when not specified\n   One run may support several criteria; multiple runs may support one\n   criterion. For each evidence item, use the most specific node or assertion\n   ID available and state what it proves as `Asserted that \u2026`. Omit optional\n   `--node` only when there is no specific ID.\n   A performance run can prove a criterion too, and one criterion can mix\n   kinds: for a query change, attach a Workflow run for the behavior and a\n   performance run for the speed on the same criterion number. Run it at the pushed head SHA\n   and attach it with `--perf-run <prun_...>`; one criterion can carry several\n   runs, and every attached item must pass. A performance run must be completed,\n   compare different base and head commits, and be at the commit you attach; a\n   slower result is refused. Attach a new run at the current head, then detach\n   an older stale item when the criterion should become proved again.\n   [cli >= 0.54.0]\n   An agent eval run can prove a criterion as well (a change to an agent\'s\n   behavior): attach it with `--eval-run <aerun_...>`. It must be a full run\n   (every case at its repeats), completed, passing its pass bar, and measured at\n   the commit you attach; a partial, failing or other-commit run is refused.\n   One criterion can carry any number of eval runs, each with its own model, and\n   every one must pass.\n   [/cli]\n\n7. After the draft PR exists, attach evidence. The server refuses a new\n   criterion number without `--text`; reuse the same text for additional\n   evidence on that number. `--detail` is no longer needed or shown. The same\n   product language applies to `--text` and `--asserted` here (step 6):\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --test <wf_...> --run <wrun_...> [--node <most-specific-id>] [--base <sha>] --asserted "Asserted that \u2026" --org <org>\n   For performance evidence, use `--perf-run` instead of `--test` and `--run`:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --perf-run <prun_...> --asserted "Asserted that \u2026" --org <org>\n   [cli >= 0.54.0]\n   For an eval run, use `--eval-run` in the same place:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_...> --asserted "Asserted that \u2026" --org <org>\n   [/cli]\n   An approved Workflow that is not a fork is attached the same way; the\n   result says `"existing": true`.\n   [cli >= 0.56.0]\n   When `--text`, `--asserted`, `--notes` or `--not-proved` contains something\n   that looks like a code identifier or an id (a name like `triageEnabled=false`,\n   a UUID, a `wf_\u2026` or `wrun_\u2026` id, or a commit SHA), attach still records the\n   evidence and prints a `warning:` line naming the token. Reword the text in\n   product language and attach again.\n   [/cli]\n   Attach exactly one evidence subject per command. A test containing any\n   `visual_checkpoint` node is a UI test, including a test that also has other\n   steps. Attaching it runs the same test at the PR base commit and the proof\n   page shows each checkpoint\'s base, head, and diff. When `--base` is omitted,\n   the CLI uses `git merge-base HEAD origin/<default>` when that ref is available.\n   Put checkpoints before\n   steps that depend on the new behavior so the base run can reach them.\n\n8. Push first, then render the criteria block:\n     cofactor proofs render --pr <n> --org <org>\n   The output already includes the markers. Paste the entire output into the\n   PR description, verbatim, replacing any existing proof block. Never\n   hand-format or edit it, and do not add a prose "Proof notes" section that\n   restates it. It looks like this:\n\n     <!-- cofactor:proof -->\n\n     **Proof of acceptance criteria** \xB7 [View all proofs in Cofactor](<proof page>)\n\n     \u{1F7E2} 2 proved (1 stale) \xB7 \u26AA 1 checked another way \xB7 \u{1F7E1} 1 not proved \xB7 as of [`a1b2c3d`](<commit page>)\n\n     | Status | # | Criterion | Tested | Asserted | Link |\n     | --- | --- | --- | --- | --- | --- |\n     | \u{1F7E2} Proved by Cofactor | 1 | The saved filter stays selected after a reload | Filter bar keeps its selection<br>E2E run in a sandbox | Asserted that the Status filter still shows "Open" after a reload. | [View the run](<run page>) |\n     | \u{1F7E2} Proved by Cofactor<br>Stale: proved at [`9f8e7d6`](<commit page>) | 2 | The list query is no slower | Performance run (prod) | Asserted that the list query is no slower than base. | [See the comparison](<perf run page>) |\n     | \u26AA Checked another way | 3 | The desktop client shows the badge | Checked another way | Opened the desktop build and checked the badge. | [See the proof](<proof page>) |\n     | \u{1F7E1} Not proved | 4 | Very large exports finish |   | No data set is large enough here; the nightly load job covers it. | [See the proof](<proof page>) |\n\n     _\u{1F7E2} proved by Cofactor (stale: proved at an earlier commit) \xB7 \u26AA checked another way \xB7 \u{1F7E1} not proved, with the reason given \xB7 \u{1F534} failed. This block is a snapshot; the live verdict is the `cofactor / proofs` check._\n\n     <!-- /cofactor:proof -->\n\n   The block is one table, and every criterion is in exactly one state:\n   \u{1F7E2} proved by Cofactor (a passing run, visual diff, perf or eval; "Stale"\n   means it was proved at an earlier commit), \u26AA checked another way (you say\n   how, with links), \u{1F7E1} not proved (you say why), or \u{1F534} failed. The check\n   passes on proved and checked another way; not proved is a warning, not a\n   failure, so say so honestly. Each evidence item is a row naming the test\n   (its user goal), how it ran, what it asserted, and a link to it; a\n   criterion with several items continues on rows with blank criterion cells.\n   The block is a snapshot as of the\n   commit in its header: it goes stale after a push, and the live verdict is\n   the `cofactor / proofs` check and the proof page. After every push, attach,\n   or detach, render again and replace the block. Context for a reviewer goes\n   in the `Asserted that \u2026` line, not in a paragraph in the PR body.\n\n9. Inspect or revise the proof:\n     cofactor proofs status --repo <owner/repo> --pr <n> --org <org>\n   The repo is inferred from `GITHUB_REPOSITORY` or `git remote get-url origin`\n   and can be overridden with `--repo`.\n   Detaching removes a criterion and all evidence attached to it:\n     cofactor proofs detach --pr <n> --criterion <n> --org <org>\n   Remove just one item with `--test <wf_...>`, `--perf-run <prun_...>` or\n   `--manual`:\n     cofactor proofs detach --pr <n> --criterion <n> --perf-run <prun_...> --org <org>\n   [cli >= 0.54.0]\n   `--eval-run <aerun_...>` removes one eval run the same way.\n   [/cli]\n   Render again and replace the PR proof block after every attach or detach.\n\nSTEPS A REVIEWER CAN FOLLOW\n\n[cli >= 0.56.0]\nWhen you attach a Workflow run, also pass `--steps`: what the test does, in\nplain sentences, one per line, written by you from the run you watched. The\nproof page shows them next to the video instead of the test\'s own step list,\nso a reviewer can follow the run without reading the test. One step per action\nor check, in the order it happened, in the language of the product (see step\n6):\n     cofactor proofs attach ... --steps @steps.txt --org <org>\n   where steps.txt holds:\n     Opened the cart\n     Added two items\n     Checked the total reads $40\n   Pass the text directly instead of a file if it is short. Attaching again\n   replaces the evidence, steps included, so pass `--steps` again whenever you\n   re-attach a run. A re-run from the proof page keeps your steps. Leave\n   `--steps` off only for an old CLI; evidence without steps falls back to the\n   test\'s own list.\n[/cli]\n\nWHEN COFACTOR CANNOT PROVE A CRITERION\n\nA criterion Cofactor cannot prove is either checked another way or not proved.\nChecked another way (manual evidence) is for a criterion Cofactor cannot exercise at all (for example\nit needs a desktop client, real hardware or a third-party console). If the\ncriterion is plainly outside a browser or a sandbox, go straight to it;\notherwise try a Cofactor test first. A failing, flaky or hard-to-write test is\nnever a reason for manual evidence: fix the test or the change, and never detach\nfailing evidence to replace it with manual evidence.\n\nSay what you did and why Cofactor could not prove it, from a clean checkout at\nthe pushed commit:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --manual --notes <text|@file> [--artifact <https-url>]... --org <org>\n   Notes (up to 10,000 characters) and links (up to 20) are shown to everyone\n   who can read the proof: keep notes to what you did and why, and redact\n   secrets, tokens and customer data from anything you link. Links must be\n   https URLs reviewers can open, never signed or tokenized links and never\n   with credentials (an image attached to the pull request works; where your\n   organization has Cofactor utilities, `cofactor utilities object upload\n   <file> --public --json` returns one).\n   A criterion holds one manual item, and attaching again replaces its notes\n   and links, so pass the same `--notes` and every `--artifact` again. A\n   criterion whose only evidence is manual passes the check with `\u{1F4DD}` and\n   `[manual]`, renders \u26AA Checked another way in the block, and shows your notes\n   on the proof page; a criterion that mixes manual with other kinds\n   keeps its stoplight and is decided by its other items. Manual evidence never promotes\n   anything and reads stale after a push; attach it again at the new head.\n   [cli >= 0.56.0]\n   When you did not check a criterion at all, or could not, say why instead of\n   leaving it out or inventing evidence:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --not-proved "<why it is not proved>" --org <org>\n   It renders \u{1F7E1} Not proved with your reason and is a warning on the check,\n   never a failure. Use it only on a criterion with no evidence (detach\n   evidence first); attaching evidence to the criterion later replaces it.\n   Write the reason in product language, like the criterion.\n   [/cli]\n\nCHECKS AND STALE EVIDENCE\n\nThe App publishes `cofactor / proofs`; the Actions job reports the same server\nverdict. Diagnose with a body file:\n     cofactor proofs check --pr <n> --sha <head> --body-file <file> --org <org>\n   Or pass `--body -` for stdin. The App queues a refreshed check after attach\nor detach. In the Actions path, rerun the check job after attaching or\ndetaching. Successful stale evidence remains successful and is\nmarked stale. Use Re-run on the Cofactor proof page for stale E2E evidence at\nthe PR head. For stale performance evidence, run it at the new head, attach\nthat run and detach the old item. Cofactor does not rerun automatically. Server and CI own promotion\ntriggers; `cofactor trigger` is not your job.\n\nNo block means no proof check; a block with no criteria fails. When the pull\nrequest has no observable acceptance criterion at all (for example a\ndependency bump or a docs-only change), print the neutral block and paste it\nin place of any proof block:\n     cofactor proofs render --none "<why this pull request has no criterion to prove>"\nIt reports a neutral result. A criterion that only Cofactor cannot exercise\ngets manual evidence instead (above).\n\nCURATING THE SUITE\n\nEvery promoted test runs from then on, so keep the suite worth running:\n   - Reuse first: attach the run of an existing test or eval that already proves\n     the criterion before creating one.\n   - Extend an existing test instead of duplicating its journey with a new one.\n   - One test per user goal, named for that goal (what the user accomplishes,\n     not the component or the ticket).\n   - Promote lasting behaviour whose regression would matter.\n   - Set a Workflow to `--promote-on never` when its proof is about the change\n     itself: a one-time migration, backfill, redirect or flag rollout step,\n     behaviour about to be removed, or a one-off visual before/after of a\n     redesign.\n   - Perf and eval runs never join a suite through proofs: a perf A/B or an\n     eval experiment for one decision stays a run. Save a perf test only for a\n     hot path (see the perf guide).\n   - Don\'t keep a test nobody would miss if it disappeared.\n\nWHAT GOES IN THE PULL REQUEST\n\nPaste the output of `proofs render` and nothing that restates it: no "Proof\nnotes" section. If a reviewer needs to know why a Workflow was forked or what\nstays uncovered, say it once in the `Asserted that \u2026` line, or make it a\ncriterion that is checked another way or not proved.\n\nUse neutral example names and never include customer names, domains, or\ncredentials. A human approves Workflows by merging the pull request; do not\napprove them yourself.\n';
 
 // ../content/src/proofs-guide/index.ts
 var PROOFS_GUIDE_SECTIONS = [
@@ -1701,7 +1701,7 @@ var PERF_GUIDE_SECTIONS = [
 ];
 
 // ../content/src/eval-guide/overview.txt
-var overview_default6 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 3                      # attempts per case, 1 to 7\n  passAtLeast: 2                  # a case passes when this many attempts pass\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n';
+var overview_default6 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 1                      # attempts per case, 1 to 7 (default 1; for a stochastic agent raise it with passAtLeast)\n  passAtLeast: 1                  # a case passes when this many attempts pass (default 1, at most repeats)\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n[cli >= 0.56.0]\n- A wrong label or input is corrected with a new version, not by deleting the\n  case: `cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n  --expected <file|-> --source "<provenance>" --replace --reason "<why>" --org <org>`\n  stores version N+1 with who, when and the reason. Without `--replace` an\n  existing key is still an error. Runs that already used the case keep the\n  version they graded and show it as `key (vN)`; new runs use the latest.\n  `cofactor eval compare` refuses runs that graded different case versions, so\n  re-run the baseline after a correction.\n- `cofactor eval case <name> <key> [--version N]` prints a case\'s expected\n  label, provenance and version history (newest first).\n[/cli]\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. With the defaults (one\nrepeat, one to pass) a case passes when its single attempt passes. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n';
 
 // ../content/src/eval-guide/index.ts
 var EVAL_GUIDE_SECTIONS = [
@@ -1970,4 +1970,4 @@ export {
   shouldBrandConsoleOutput,
   installConsoleBranding
 };
-//# sourceMappingURL=chunk-KL2ARGH6.js.map
+//# sourceMappingURL=chunk-2OHYMT3A.js.map