@runcofactor/cli 0.54.0 → 0.55.0

package/dist/{chunk-KNWKOKM7.js → chunk-KL2ARGH6.js}

@@ -1656,11 +1656,19 @@ var EXTERNAL_RUN_GUIDE_SECTIONS = [
   { title: "Write an evidence-backed report", slug: "reporting", content: reporting_default }
 ];
 
+// ../content/src/exploratory-test-guide/overview.txt
+var overview_default2 = 'Start an exploratory test: a Cofactor agent drives a real browser against your\napplication from a free-text prompt, explores, and reports what it finds. Use it\nwhen you want an agent to explore (try a flow, look for problems) instead of\ndriving the browser yourself with `cofactor session guide`.\n\nThis is not `cofactor exploration`, which runs product-map cycles, and it is not\na regression test. To write a repeatable test, read\n`cofactor workflow guide write-a-test`.\n\nSTART ONE\n\n  cofactor exploratory-test run --application <name|id> --prompt "Try the checkout flow as a new customer" --wait\n  cofactor exploratory-test run --application <name|id> --prompt @charter.md --wait\n  cofactor exploratory-test run --application <name|id> --prompt @charter.md --sandbox-env <name> --wait\n\n  --prompt takes text or @file (up to 5,000 characters). Name the goal, the\n  role the agent acts as and what a problem looks like. --wait polls until the\n  run finishes, prints the status and URL, and exits non-zero unless it\n  completed. Add --timeout <seconds> to change the 1800 second limit.\n\nCHOOSE THE TARGET\n\n  With no target flag the application\'s default ad-hoc environment is used,\n  else the live one.\n  --environment <name|id>   a named live environment of the application\n  --credential <name|id>    sign in with this credential (live targets)\n  --sandbox-env <name>      a Sandboxes v2 environment of the application.\n                            Personas come from the environment, so do not\n                            combine it with --environment or --credential.\n\nCHOOSE THE FLAVOR\n\n  --flavor functional (default) | ux | spec | fuzz\n  --effort quick (about 5 minutes) | standard (about 15, default) | deep (about 30)\n\n  --effort applies to fuzz only. fuzz needs --sandbox-env. ux and spec run on\n  live environments only, so do not combine them with --sandbox-env.\n\nVARIABLES\n\n  A functional run on a live environment can use its variables\n  (`cofactor env set-var`): name one in the prompt as $var.ENV.<name> and the\n  agent types it by reference. A sensitive value is never shown to the agent: it\n  reads [REDACTED] in tool results, files and the trace. Screenshots and video\n  are not masked.\n\nREAD THE RESULT\n\n  cofactor exploratory-test get <id>\n  cofactor exploratory-test list --application <name|id>\n  cofactor exploratory-test list --flavor ux --page 2 --page-size 20\n\n  `get` shows the status, summary, issues found, whether a video and trace\n  exist, and the URL. Run `cofactor exploratory-test --help` for every option.\n  To run one on a schedule: `cofactor schedule exploratory-test --help`.\n\nRULES\n\n  - Do not put a real password or token in a prompt. Use a credential, a\n    persona or a $var.ENV.<name> reference.\n  - Use neutral example names (Acme, example.com) in prompts you save.\n';
+
+// ../content/src/exploratory-test-guide/index.ts
+var EXPLORATORY_TEST_GUIDE_SECTIONS = [
+  { title: "Exploratory tests", slug: "overview", content: overview_default2 }
+];
+
 // ../content/src/issues-guide/overview.txt
-var overview_default2 = 'Use these commands to find, read, and prioritize issues Cofactor has detected.\n\nLIST AND FILTER\n\n  cofactor issues list\n  cofactor issues list --status open --severity high --format markdown\n  cofactor issues list --search "timeout" --application-id <uuid>\n  cofactor issues list --tag checkout,regression --status open,closed\n  cofactor issues list --page 2 --page-size 50\n\n  Filters: --search, --severity (low|medium|high|unknown), --status\n  (open|closed|not_a_bug, comma-separated), --tag (any of these tag names),\n  --application-id (alias --property-id). Add --json for compact JSON, or\n  --json --full for the unmodified response.\n\nREAD ONE ISSUE\n\n  cofactor issues get <issueId>\n  cofactor issues get <issueId> --format markdown\n  cofactor issues get <issueId> --json --include console,network,repro,diagnostic-full\n  cofactor issues get <issueId> --json --download ./evidence\n\n  The detail view carries the diagnosis, the latest occurrence (URL, console\n  and network failures, repro steps), occurrence count, first and last seen\n  times, assignee, and the workflow run that surfaced it. Note the timestamps,\n  URLs, error text, and HTTP status codes; you need them to confirm a root cause.\n\n  It also lists the evidence items of each of the 10 newest occurrences,\n  grouped by occurrence. Evidence stays on the occurrence that produced it, so\n  a later sighting often has none while an earlier, diagnosed one does. Each\n  item has a number, kind, description, optional detail, and a signed link\n  (valid about an hour; do not paste it publicly) for each screenshot or clip.\n  --download <dir> saves those files as <occurrenceId>-evidence-<n>-frame.jpg,\n  -clip.mp4 and so on, so you can open the screenshots and look at them.\n\nTRIAGE ORDER\n\n  1. Start with open issues, severity high first, then by occurrence count and\n     most recent last-seen time. A recurring issue outranks a one-off.\n  2. Open each candidate and read the diagnosis before judging it.\n  3. Decide one of: real and new (keep open, tag it); duplicate (merge it);\n     intended behavior or noise (close as not-a-bug with a reason); fixed (close).\n  4. See `dedupe-and-close` for the commands, `logs` to find root cause, and\n     `handoff` to pass a confirmed bug to someone who will fix it.\n\nFILE A NEW ISSUE\n\n  Search first so you do not create a duplicate:\n    cofactor issues create --check --title "<title>" --application-id <uuid>\n  Then:\n    cofactor issues create --title "<title>" --application-id <uuid> \\\n      --url <page-url> --severity medium --body "<markdown>"\n  Link evidence with --from-run <runId>, --code-ref <path[:line]>,\n  --sentry <id|url>, or --log-query <query>. Add your own evidence items with\n  --evidence <kind>:<description>, each followed by an optional\n  --detail <text> and --screenshot <file> (uploaded and shown with the item):\n    cofactor issues bump <id> --url <page-url> \\\n      --evidence "visual:The Pay button stays disabled" --screenshot ./pay.png\n\nEDIT AN ISSUE\n\n  cofactor issues update <id> --summary "<markdown>" --repro-steps "<steps>"\n  cofactor issues update <id> --occurrence <occurrenceId> \\\n    --replace-evidence "1:visual:<corrected description>" --screenshot ./shot.png \\\n    --remove-evidence 2 --evidence "timeline_event:<what happened>"\n\n  Edits apply to the latest occurrence unless --occurrence names another;\n  `issues get` lists evidence by occurrence with the numbers these flags use.\n  A replaced item is replaced whole, so pass --screenshot again to keep one.\n  Evidence cannot be edited while the occurrence\'s diagnosis is running or\n  after it failed, and re-running a diagnosis replaces hand-edited evidence.\n  Every edit is recorded in the issue\'s audit history.\n\nLINK AN ISSUE TO THE PRODUCT MAP\n\n  cofactor issues link <id> --capability CAP-FLW-003\n  cofactor issues link <id> --capability CAP-FLW-003 --rule prule_<id>\n  cofactor issues unlink <id> --capability CAP-FLW-003 [--rule prule_<id>]\n  cofactor issues create ... --capability CAP-FLW-003 [--rule prule_<id>]\n\n  A link says which capability (and, with --rule, which of its rules) the bug\n  breaks. Codes resolve within the issue\'s application; rule ids come from\n  `cofactor product-map show <code>`. `issues get` lists the links, and\n  `product-map show` lists the open issues on the capability and under each\n  rule. Link only behaviour the bug actually breaks: an open linked issue is\n  what `product-map coverage` reads to show a capability as failing on a\n  known issue (its flows red) or blocked by the issue (no flow yet), and a\n  linked issue that closes flags the capability\'s flows for a rerun.\n';
+var overview_default3 = 'Use these commands to find, read, and prioritize issues Cofactor has detected.\n\nLIST AND FILTER\n\n  cofactor issues list\n  cofactor issues list --status open --severity high --format markdown\n  cofactor issues list --search "timeout" --application-id <uuid>\n  cofactor issues list --tag checkout,regression --status open,closed\n  cofactor issues list --page 2 --page-size 50\n\n  Filters: --search, --severity (low|medium|high|unknown), --status\n  (open|closed|not_a_bug, comma-separated), --tag (any of these tag names),\n  --application-id (alias --property-id). Add --json for compact JSON, or\n  --json --full for the unmodified response.\n\nREAD ONE ISSUE\n\n  cofactor issues get <issueId>\n  cofactor issues get <issueId> --format markdown\n  cofactor issues get <issueId> --json --include console,network,repro,diagnostic-full\n  cofactor issues get <issueId> --json --download ./evidence\n\n  The detail view carries the diagnosis, the latest occurrence (URL, console\n  and network failures, repro steps), occurrence count, first and last seen\n  times, assignee, and the workflow run that surfaced it. Note the timestamps,\n  URLs, error text, and HTTP status codes; you need them to confirm a root cause.\n\n  It also lists the evidence items of each of the 10 newest occurrences,\n  grouped by occurrence. Evidence stays on the occurrence that produced it, so\n  a later sighting often has none while an earlier, diagnosed one does. Each\n  item has a number, kind, description, optional detail, and a signed link\n  (valid about an hour; do not paste it publicly) for each screenshot or clip.\n  --download <dir> saves those files as <occurrenceId>-evidence-<n>-frame.jpg,\n  -clip.mp4 and so on, so you can open the screenshots and look at them.\n\nTRIAGE ORDER\n\n  1. Start with open issues, severity high first, then by occurrence count and\n     most recent last-seen time. A recurring issue outranks a one-off.\n  2. Open each candidate and read the diagnosis before judging it.\n  3. Decide one of: real and new (keep open, tag it); duplicate (merge it);\n     intended behavior or noise (close as not-a-bug with a reason); fixed (close).\n  4. See `dedupe-and-close` for the commands, `logs` to find root cause, and\n     `handoff` to pass a confirmed bug to someone who will fix it.\n\nFILE A NEW ISSUE\n\n  Search first so you do not create a duplicate:\n    cofactor issues create --check --title "<title>" --application-id <uuid>\n  Then:\n    cofactor issues create --title "<title>" --application-id <uuid> \\\n      --url <page-url> --severity medium --body "<markdown>"\n  Link evidence with --from-run <runId>, --code-ref <path[:line]>,\n  --sentry <id|url>, or --log-query <query>. Add your own evidence items with\n  --evidence <kind>:<description>, each followed by an optional\n  --detail <text> and --screenshot <file> (uploaded and shown with the item):\n    cofactor issues bump <id> --url <page-url> \\\n      --evidence "visual:The Pay button stays disabled" --screenshot ./pay.png\n\nEDIT AN ISSUE\n\n  cofactor issues update <id> --summary "<markdown>" --repro-steps "<steps>"\n  cofactor issues update <id> --occurrence <occurrenceId> \\\n    --replace-evidence "1:visual:<corrected description>" --screenshot ./shot.png \\\n    --remove-evidence 2 --evidence "timeline_event:<what happened>"\n\n  Edits apply to the latest occurrence unless --occurrence names another;\n  `issues get` lists evidence by occurrence with the numbers these flags use.\n  A replaced item is replaced whole, so pass --screenshot again to keep one.\n  Evidence cannot be edited while the occurrence\'s diagnosis is running or\n  after it failed, and re-running a diagnosis replaces hand-edited evidence.\n  Every edit is recorded in the issue\'s audit history.\n\nLINK AN ISSUE TO THE PRODUCT MAP\n\n  cofactor issues link <id> --capability CAP-FLW-003\n  cofactor issues link <id> --capability CAP-FLW-003 --rule prule_<id>\n  cofactor issues unlink <id> --capability CAP-FLW-003 [--rule prule_<id>]\n  cofactor issues create ... --capability CAP-FLW-003 [--rule prule_<id>]\n\n  A link says which capability (and, with --rule, which of its rules) the bug\n  breaks. Codes resolve within the issue\'s application; rule ids come from\n  `cofactor product-map show <code>`. `issues get` lists the links, and\n  `product-map show` lists the open issues on the capability and under each\n  rule. Link only behaviour the bug actually breaks: an open linked issue is\n  what `product-map coverage` reads to show a capability as failing on a\n  known issue (its flows red) or blocked by the issue (no flow yet), and a\n  linked issue that closes flags the capability\'s flows for a rerun.\n';
 
 // ../content/src/issues-guide/dedupe-and-close.txt
-var dedupe_and_close_default = 'Keep the issue list honest: one issue per real problem, closed when resolved.\n\nCLOSE\n\n  cofactor issues close <issueId>\n  cofactor issues close <issueId> --not-a-bug --reason "<why this is intended>"\n\n  --not-a-bug requires --reason. Cofactor records the reason as intended\n  behavior so the same behavior is not reported again; write it as a clear\n  statement of the rule ("Archived projects intentionally hide the Edit\n  button"), not as an apology.\n\nREOPEN\n\n  cofactor issues reopen <issueId>\n\n  Use when a closed issue recurs. This also clears a duplicate link.\n\nMERGE DUPLICATES\n\n  cofactor issues merge <dupId>[,<dupId>...] --into <canonicalId>\n\n  The duplicates are closed, their tags are copied to the canonical issue, it\n  gains a comment, and later sightings land on the canonical issue. Pick the\n  oldest or best-documented issue as canonical. Confirm they share a root\n  cause, not just a symptom, before merging.\n\nADD A SIGHTING\n\n  cofactor issues bump <issueId> --url <page-url> [--from-run <runId>] [--body "<note>"]\n\n  Records that the problem was seen again, attaching the run\'s video and trace,\n  instead of filing a duplicate.\n\nTAG\n\n  cofactor issues tag <issueId> --add regression,checkout\n  cofactor issues tag <issueId> --remove checkout\n  cofactor issues tag <issueId> --set regression\n\n  Tags are referenced by name. Create a missing tag first:\n    cofactor tag create --name <name> --color <color>\n\nRULES\n\n  - Search (`issues list --search`, or `issues create --check`) before filing\n    or closing.\n  - Never close as not-a-bug without a reason you could defend to the person\n    who owns the feature.\n  - Add --json to any command for scripting.\n';
+var dedupe_and_close_default = 'Keep the issue list honest: one issue per real problem, closed when resolved.\n\nCLOSE\n\n  cofactor issues close <issueId>\n  cofactor issues close <issueId> --not-a-bug --reason "<why this is intended>"\n\n  --not-a-bug requires --reason. Cofactor records the reason as intended\n  behavior so the same behavior is not reported again; write it as a clear\n  statement of the rule ("Archived projects intentionally hide the Edit\n  button"), not as an apology.\n\nREOPEN\n\n  cofactor issues reopen <issueId>\n\n  Use when a closed issue recurs. This also clears a duplicate link.\n\nMERGE DUPLICATES\n\n  cofactor issues merge <dupId>[,<dupId>...] --into <canonicalId>\n\n  The duplicates are closed, their tags are copied to the canonical issue, it\n  gains a comment, and later sightings land on the canonical issue. Pick the\n  oldest or best-documented issue as canonical. Confirm they share a root\n  cause, not just a symptom, before merging.\n\nADD A SIGHTING\n\n  cofactor issues bump <issueId> --url <page-url> [--from-run <runId>] [--body "<note>"]\n\n  Records that the problem was seen again, attaching the run\'s video and trace,\n  instead of filing a duplicate.\n\nCOMMENT\n\n  cofactor issues comment <issueId> --body "<text>"\n  cofactor issues comment <issueId> --body @note.md\n\n  Leaves a note for the team on the issue. `issues get` lists the comments.\n\nTAG\n\n  cofactor issues tag <issueId> --add regression,checkout\n  cofactor issues tag <issueId> --remove checkout\n  cofactor issues tag <issueId> --set regression\n\n  Tags are referenced by name. Create a missing tag first:\n    cofactor tag create --name <name> --color <color>\n\nRULES\n\n  - Read an issue with `issues get` before closing or merging it.\n  - Never paste credentials or tokens from an issue\'s evidence into tickets,\n    comments or chat.\n  - Search (`issues list --search`, or `issues create --check`) before filing\n    or closing.\n  - Never close as not-a-bug without a reason you could defend to the person\n    who owns the feature.\n  - Add --json to any command for scripting.\n';
 
 // ../content/src/issues-guide/logs.txt
 var logs_default = 'Cross-reference an issue with your own server logs to find the root cause.\nCofactor sees the symptom in the browser; your logs show the cause.\n\n1. Get the evidence from the issue\n     cofactor issues get <issueId> --json --include console,network,repro\n   Write down: when it was detected (timestamp), the URL and API routes\n   involved, visible error text, failed requests and their HTTP status codes,\n   and any request or correlation IDs.\n\n2. Query your logs for the same window\n   Use whatever your team uses (CloudWatch, Datadog, kubectl logs, a local log\n   file, Sentry). Search a few seconds either side of the detection time,\n   filtered by the route and the error text. Cofactor does not hold your server\n   logs; ask the user where they live if you do not know.\n\n3. Correlate\n   Match on timestamp, error message, HTTP status (4xx/5xx for the failed\n   request), URL or route, stack trace, and request ID. A request ID lets you\n   follow one request end to end.\n\n4. Report\n   - State match confidence: clear, likely, or circumstantial.\n   - Lay out a single timeline across the browser evidence and the logs.\n   - Separate the symptom (what Cofactor caught) from the cause (what logs show).\n   - Point at the specific code, service, or config to fix.\n   - Say plainly when logs are missing or incomplete; do not speculate.\n\n5. Record what you found\n     cofactor issues create --check ...        (file only if no issue exists)\n     cofactor issues bump <issueId> --url <url> --log-query "<query>" --log-system cloudwatch\n   Attach the query with --log-query (repeatable; --log-system is one of\n   sentry, cloudwatch, datadog, other), and --sentry <id|url> and\n   --code-ref <path[:line]> for a Sentry issue or the defective code location.\n';
@@ -1670,38 +1678,38 @@ var handoff_default = 'Hand a confirmed bug to whoever will fix it, with everyth
 
 // ../content/src/issues-guide/index.ts
 var ISSUES_GUIDE_SECTIONS = [
-  { title: "Find and triage issues", slug: "overview", content: overview_default2 },
+  { title: "Find and triage issues", slug: "overview", content: overview_default3 },
   { title: "Dedupe, close, and tag", slug: "dedupe-and-close", content: dedupe_and_close_default },
   { title: "Cross-reference an issue with server logs", slug: "logs", content: logs_default },
   { title: "Hand an issue off for a fix", slug: "handoff", content: handoff_default }
 ];
 
 // ../content/src/proofs-guide/overview.txt
-var overview_default3 = 'PROVE A CHANGE WITH TESTS\n\nUse this guide for a pull request that changes user-visible behavior. The pull\nrequest connects numbered acceptance criteria to passing Workflow evidence.\nPass `--org <org>` on every write to select the organization.\n\nCHOOSING PROOFS\n\nPick the proof that fits the change; one change can need several.\n\n- Functional change: a Workflow run that passes at the pushed commit. The\n  procedure below covers it.\n- Performance-sensitive change: a perf run. Read `cofactor perf guide`. A\n  query change, for example, needs both a functional proof and a perf proof.\n- UI change: a Workflow with `visual_checkpoint` nodes. Read\n  `cofactor workflow guide visual-checkpoints`, and step 7 below for how\n  attaching runs the test at the PR base and shows the checkpoint diffs.\n[cli >= 0.54.0]\n- Agent behavior: evals. Read `cofactor eval guide`, then attach a full passing\n  eval run with `--eval-run` (see step 6).\n[/cli]\n\nTHE PROCEDURE\n\n1. Find coverage before writing a Workflow:\n     cofactor workflow list --search "<words>"\n     cofactor workflow get <wf_...> --format markdown\n   Read the candidate assertions. If a test already walks the journey, use it.\n   Never add near-duplicates or a new test for a changed filter, column or field.\n\n2. Choose the Workflow:\n   - If an approved Workflow already proves the criterion without changes (the\n     journey must still work, for example after a query or refactor change),\n     run it as is and attach its run. It is cited as an existing test: it\n     proves the journey still works, it is not new coverage, and it is never\n     promoted (it is already in its suites). It cannot prove new behavior: a\n     new filter, column or field needs a fork that asserts it. An approved\n     fork is still treated as a fork; to cite an existing test, attach the\n     approved test that is not a fork.\n   - Fork an approved Workflow with\n     `cofactor workflow fork fork <wf_...> --org <org>` and edit the fork.\n     Never edit the approved original.\n   - Create a new draft only when no existing journey fits or it needs its own\n     data or role (see `cofactor workflow guide write-a-test`).\n\n3. Configure promotion on the fork or new draft before attaching evidence:\n     cofactor workflow set-promotion <workflowId> --promote-on merge|never|deploy:<stage> [--target-suite <suite>]... --org <org>\n   Repeat `--target-suite` for multiple suites. The command replaces the target\n   suite list; omit it to clear extra suites. Promotion adds target suites and\n   the default regression suite when present. An org without one can use the\n   **Create default suite** button. Do not choose a layer.\n   `never` archives the test at merge instead of adding it to any suite, and\n   takes no `--target-suite`; use it when the proof is about the change itself\n   (see CURATING THE SUITE). It has no effect on an already-approved test.\n\n4. Prove the change at the exact pushed commit. Start from a clean checkout,\n   commit and push the application change, then use the pushed commit from\n   `git rev-parse HEAD` as `<sha>` in `--at <sha>`. Claim that SHA and sync it:\n     cofactor sandbox claim --application <app> --at <sha> --wait --org <org>\n     cofactor sandbox sync <sandboxId> [--build] --org <org>\n     cofactor workflow run <workflowId> --sandbox <sandboxId> --stream --org <org>\n   Iterate until it passes, then run it once more after the last application or\n   Workflow test change and use that passing run ID. An existing test is run the\n   same way, at the pushed commit and on its current revision. For application edits,\n   commit and push, then sync and rerun. Leave the checkout clean. Attach\n   refuses dirty checkouts or missing commit evidence. If the run records a\n   synced tree, it must equal `HEAD^{tree}`; otherwise its recorded source SHA\n   must equal `HEAD`. The run must exist, belong to the cited Workflow, pass,\n   and use its current revision. Attach requires a GitHub-delivered application;\n   upload-delivered applications cannot supply this proof.\n\n5. After proving, commit any remaining changes and push the branch. If those\n   commits change the application or Workflow tests, return to step 4 and\n   prove them before proceeding. Open a DRAFT pull request and note its number\n   `<n>`; attach needs that PR number.\n\n6. Write concise numbered acceptance criteria. State observable outcomes\n   plainly. One run may support several criteria; multiple runs may support one\n   criterion. For each evidence item, use the most specific node or assertion\n   ID available and state what it proves as `Asserted that \u2026`. Omit optional\n   `--node` only when there is no specific ID.\n   A performance run can prove a criterion too, and one criterion can mix\n   kinds: for a query change, attach a Workflow run for the behavior and a\n   performance run for the speed on the same criterion number. Run it at the pushed head SHA\n   and attach it with `--perf-run <prun_...>`; one criterion can carry several\n   runs, and every attached item must pass. A performance run must be completed,\n   compare different base and head commits, and be at the commit you attach; a\n   slower result is refused. Attach a new run at the current head, then detach\n   an older stale item when the criterion should become proved again.\n   [cli >= 0.54.0]\n   An agent eval run can prove a criterion as well (a change to an agent\'s\n   behavior): attach it with `--eval-run <aerun_...>`. It must be a full run\n   (every case at its repeats), completed, passing its pass bar, and measured at\n   the commit you attach; a partial, failing or other-commit run is refused.\n   One criterion can carry any number of eval runs, each with its own model, and\n   every one must pass.\n   [/cli]\n\n7. After the draft PR exists, attach evidence. The server refuses a new\n   criterion number without both `--text` and `--detail`; reuse the same text\n   and detail for additional evidence on that number:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --test <wf_...> --run <wrun_...> [--node <most-specific-id>] [--base <sha>] --asserted "Asserted that \u2026" --org <org>\n   For performance evidence, use `--perf-run` instead of `--test` and `--run`:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --perf-run <prun_...> --asserted "Asserted that \u2026" --org <org>\n   [cli >= 0.54.0]\n   For an eval run, use `--eval-run` in the same place:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_...> --asserted "Asserted that \u2026" --org <org>\n   [/cli]\n   An approved Workflow that is not a fork is attached the same way; the\n   result says `"existing": true`.\n   Attach exactly one evidence subject per command. A test containing any\n   `visual_checkpoint` node is a UI test, including a test that also has other\n   steps. Attaching it runs the same test at the PR base commit and the proof\n   page shows each checkpoint\'s base, head, and diff. When `--base` is omitted,\n   the CLI uses `git merge-base HEAD origin/<default>` when that ref is available.\n   Put checkpoints before\n   steps that depend on the new behavior so the base run can reach them.\n\n8. Push first, then render the criteria block:\n     cofactor proofs render --pr <n> --org <org>\n   The output already includes the markers. Paste the entire output into the\n   PR description, verbatim, replacing any existing proof block. Never\n   hand-format or edit it, and do not add a prose "Proof notes" section that\n   restates it. It looks like this:\n\n     <!-- cofactor:proof -->\n\n     **Proof of acceptance criteria** \xB7 [View all proofs in Cofactor](<proof page>)\n\n     \u{1F7E2} 1 passed \xB7 \u{1F7E1} 1 stale \xB7 \u26AA 1 manual \xB7 as of `a1b2c3d`\n\n     \u{1F7E2} **1. The saved filter stays selected after a reload** \xB7 Passed\n     - **Tested:** Filter bar keeps its selection \xB7 E2E run in a sandbox \xB7 [View the run](<run page>)\n       - Asserted that the Status filter still shows "Open" after a reload.\n\n     \u{1F7E1} **2. The list query is no slower** \xB7 Stale (proved at 9f8e7d6)\n     - **Tested:** Performance run (prod) \xB7 [See the comparison](<perf run page>)\n       - Asserted that the list query is no slower than base.\n\n     \u26AA **3. The desktop client shows the badge** \xB7 Manual\n     - **Tested:** Manual check, not provable with Cofactor \xB7 [See the proof](<proof page>)\n       - Opened the desktop build and checked the badge.\n\n   Each criterion leads with a stoplight: \u{1F7E2} passed, \u{1F7E1} passed at an earlier\n   commit (stale), \u{1F534} failed, \u26AA shown manually because Cofactor could not prove\n   it. Under it, each evidence item names the test (its user goal), how it\n   ran, what it asserted, and links to it. The block is a snapshot as of the\n   commit in its header: it goes stale after a push, and the live verdict is\n   the `cofactor / proofs` check and the proof page. After every push, attach,\n   or detach, render again and replace the block. Context for a reviewer goes\n   in the criterion\'s `--detail` and in the `Asserted that \u2026` line (the proof\n   page shows both), not in a paragraph in the PR body.\n\n9. Inspect or revise the proof:\n     cofactor proofs status --repo <owner/repo> --pr <n> --org <org>\n   The repo is inferred from `GITHUB_REPOSITORY` or `git remote get-url origin`\n   and can be overridden with `--repo`.\n   Detaching removes a criterion and all evidence attached to it:\n     cofactor proofs detach --pr <n> --criterion <n> --org <org>\n   Remove just one item with `--test <wf_...>`, `--perf-run <prun_...>` or\n   `--manual`:\n     cofactor proofs detach --pr <n> --criterion <n> --perf-run <prun_...> --org <org>\n   [cli >= 0.54.0]\n   `--eval-run <aerun_...>` removes one eval run the same way.\n   [/cli]\n   Render again and replace the PR proof block after every attach or detach.\n\nWHEN COFACTOR CANNOT PROVE A CRITERION\n\nManual evidence is for a criterion Cofactor cannot exercise at all (for example\nit needs a desktop client, real hardware or a third-party console). If the\ncriterion is plainly outside a browser or a sandbox, go straight to it;\notherwise try a Cofactor test first. A failing, flaky or hard-to-write test is\nnever a reason for manual evidence: fix the test or the change, and never detach\nfailing evidence to replace it with manual evidence.\n\nSay what you did and why Cofactor could not prove it, from a clean checkout at\nthe pushed commit:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --manual --notes <text|@file> [--artifact <https-url>]... --org <org>\n   Notes (up to 10,000 characters) and links (up to 20) are shown to everyone\n   who can read the proof: keep notes to what you did and why, and redact\n   secrets, tokens and customer data from anything you link. Links must be\n   https URLs reviewers can open, never signed or tokenized links and never\n   with credentials (an image attached to the pull request works; where your\n   organization has Cofactor utilities, `cofactor utilities object upload\n   <file> --public --json` returns one).\n   A criterion holds one manual item, and attaching again replaces its notes\n   and links, so pass the same `--notes` and every `--artifact` again. A\n   criterion whose only evidence is manual passes the check with `\u{1F4DD}` and\n   `[manual]`, renders \u26AA Manual in the block, and shows a Manual badge with\n   your notes on the proof page; a criterion that mixes manual with other kinds\n   keeps its stoplight and is decided by its other items. Manual evidence never promotes\n   anything and reads stale after a push; attach it again at the new head.\n\nCHECKS AND STALE EVIDENCE\n\nThe App publishes `cofactor / proofs`; the Actions job reports the same server\nverdict. Diagnose with a body file:\n     cofactor proofs check --pr <n> --sha <head> --body-file <file> --org <org>\n   Or pass `--body -` for stdin. The App queues a refreshed check after attach\nor detach. In the Actions path, rerun the check job after attaching or\ndetaching. Successful stale evidence remains successful and is\nmarked stale. Use Re-run on the Cofactor proof page for stale E2E evidence at\nthe PR head. For stale performance evidence, run it at the new head, attach\nthat run and detach the old item. Cofactor does not rerun automatically. Server and CI own promotion\ntriggers; `cofactor trigger` is not your job.\n\nNo block means no proof check; a block with no criteria fails. When the pull\nrequest has no observable acceptance criterion at all (for example a\ndependency bump or a docs-only change), print the neutral block and paste it\nin place of any proof block:\n     cofactor proofs render --none "<why this pull request has no criterion to prove>"\nIt reports a neutral result. A criterion that only Cofactor cannot exercise\ngets manual evidence instead (above).\n\nCURATING THE SUITE\n\nEvery promoted test runs from then on, so keep the suite worth running:\n   - Reuse first: attach the run of an existing test or eval that already proves\n     the criterion before creating one.\n   - Extend an existing test instead of duplicating its journey with a new one.\n   - One test per user goal, named for that goal (what the user accomplishes,\n     not the component or the ticket).\n   - Promote lasting behaviour whose regression would matter.\n   - Set a Workflow to `--promote-on never` when its proof is about the change\n     itself: a one-time migration, backfill, redirect or flag rollout step,\n     behaviour about to be removed, or a one-off visual before/after of a\n     redesign.\n   - Perf and eval runs never join a suite through proofs: a perf A/B or an\n     eval experiment for one decision stays a run. Save a perf test only for a\n     hot path (see the perf guide).\n   - Don\'t keep a test nobody would miss if it disappeared.\n\nWHAT GOES IN THE PULL REQUEST\n\nPaste the output of `proofs render` and nothing that restates it: no "Proof\nnotes" section. If a reviewer needs to know why a Workflow was forked or what\nstays uncovered, say it once in the criterion\'s `--detail`, or make it a\ncriterion with manual evidence.\n\nUse neutral example names and never include customer names, domains, or\ncredentials. A human approves Workflows by merging the pull request; do not\napprove them yourself.\n';
+var overview_default4 = 'PROVE A CHANGE WITH TESTS\n\nUse this guide for a pull request that changes user-visible behavior. The pull\nrequest connects numbered acceptance criteria to passing Workflow evidence.\nPass `--org <org>` on every write to select the organization.\n\nCHOOSING PROOFS\n\nPick the proof that fits the change; one change can need several.\n\n- Functional change: a Workflow run that passes at the pushed commit. The\n  procedure below covers it.\n- Performance-sensitive change: a perf run. Read `cofactor perf guide`. A\n  query change, for example, needs both a functional proof and a perf proof.\n- UI change: a Workflow with `visual_checkpoint` nodes. Read\n  `cofactor workflow guide visual-checkpoints`, and step 7 below for how\n  attaching runs the test at the PR base and shows the checkpoint diffs.\n  When the organization uses visual regression (its Workflows already carry\n  `visual_checkpoint` nodes, or its admin says Visual checkpoints are on), a\n  change that alters what a screen looks like must carry UI-test evidence: run a Workflow with a `visual_checkpoint` on that screen, adding or\n  extending one if none exists. A manual screenshot does not replace it. With\n  visual regression off, skip this and say so in the criterion\'s `--detail`.\n[cli >= 0.54.0]\n- Agent behavior: evals. Read `cofactor eval guide`, then attach a full passing\n  eval run with `--eval-run` (see step 6).\n[/cli]\n\nTHE PROCEDURE\n\n1. Find coverage before writing a Workflow:\n     cofactor workflow list --search "<words>"\n     cofactor workflow get <wf_...> --format markdown\n   Read the candidate assertions. If a test already walks the journey, use it.\n   Never add near-duplicates or a new test for a changed filter, column or field.\n\n2. Choose the Workflow:\n   - If an approved Workflow already proves the criterion without changes (the\n     journey must still work, for example after a query or refactor change),\n     run it as is and attach its run. It is cited as an existing test: it\n     proves the journey still works, it is not new coverage, and it is never\n     promoted (it is already in its suites). It cannot prove new behavior: a\n     new filter, column or field needs a fork that asserts it. An approved\n     fork is still treated as a fork; to cite an existing test, attach the\n     approved test that is not a fork.\n   - Fork an approved Workflow with\n     `cofactor workflow fork fork <wf_...> --org <org>` and edit the fork.\n     Never edit the approved original.\n   - Create a new draft only when no existing journey fits or it needs its own\n     data or role (see `cofactor workflow guide write-a-test`).\n\n3. Configure promotion on the fork or new draft before attaching evidence:\n     cofactor workflow set-promotion <workflowId> --promote-on merge|never|deploy:<stage> [--target-suite <suite>]... --org <org>\n   Repeat `--target-suite` for multiple suites. The command replaces the target\n   suite list; omit it to clear extra suites. Promotion adds target suites and\n   the default regression suite when present. An org without one can use the\n   **Create default suite** button. Do not choose a layer.\n   `never` archives the test at merge instead of adding it to any suite, and\n   takes no `--target-suite`; use it when the proof is about the change itself\n   (see CURATING THE SUITE). It has no effect on an already-approved test.\n\n4. Prove the change at the exact pushed commit. Start from a clean checkout,\n   commit and push the application change, then use the pushed commit from\n   `git rev-parse HEAD` as `<sha>` in `--at <sha>`. Claim that SHA and sync it:\n     cofactor sandbox claim --application <app> --at <sha> --wait --org <org>\n     cofactor sandbox sync <sandboxId> [--build] --org <org>\n     cofactor workflow run <workflowId> --sandbox <sandboxId> --stream --org <org>\n   Keep the sandbox id. If you lose it, `cofactor sandbox claims --org <org>`\n   lists your live claims, and `cofactor sandbox destroy <sandboxId>` tears one\n   down when you are done. The run uses the claimed sandbox\'s own environment,\n   so do not pass `--environment`.\n   Iterate until it passes, then run it once more after the last application or\n   Workflow test change and use that passing run ID. An existing test is run the\n   same way, at the pushed commit and on its current revision. For application edits,\n   commit and push, then sync and rerun. Leave the checkout clean. Attach\n   refuses dirty checkouts or missing commit evidence. If the run records a\n   synced tree, it must equal `HEAD^{tree}`; otherwise its recorded source SHA\n   must equal `HEAD`. The run must exist, belong to the cited Workflow, pass,\n   and use its current revision. Attach requires a GitHub-delivered application;\n   upload-delivered applications cannot supply this proof.\n\n5. After proving, commit any remaining changes and push the branch. If those\n   commits change the application or Workflow tests, return to step 4 and\n   prove them before proceeding. Open a DRAFT pull request and note its number\n   `<n>`; attach needs that PR number.\n\n6. Write concise numbered acceptance criteria. State observable outcomes\n   plainly. One run may support several criteria; multiple runs may support one\n   criterion. For each evidence item, use the most specific node or assertion\n   ID available and state what it proves as `Asserted that \u2026`. Omit optional\n   `--node` only when there is no specific ID.\n   A performance run can prove a criterion too, and one criterion can mix\n   kinds: for a query change, attach a Workflow run for the behavior and a\n   performance run for the speed on the same criterion number. Run it at the pushed head SHA\n   and attach it with `--perf-run <prun_...>`; one criterion can carry several\n   runs, and every attached item must pass. A performance run must be completed,\n   compare different base and head commits, and be at the commit you attach; a\n   slower result is refused. Attach a new run at the current head, then detach\n   an older stale item when the criterion should become proved again.\n   [cli >= 0.54.0]\n   An agent eval run can prove a criterion as well (a change to an agent\'s\n   behavior): attach it with `--eval-run <aerun_...>`. It must be a full run\n   (every case at its repeats), completed, passing its pass bar, and measured at\n   the commit you attach; a partial, failing or other-commit run is refused.\n   One criterion can carry any number of eval runs, each with its own model, and\n   every one must pass.\n   [/cli]\n\n7. After the draft PR exists, attach evidence. The server refuses a new\n   criterion number without both `--text` and `--detail`; reuse the same text\n   and detail for additional evidence on that number:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --test <wf_...> --run <wrun_...> [--node <most-specific-id>] [--base <sha>] --asserted "Asserted that \u2026" --org <org>\n   For performance evidence, use `--perf-run` instead of `--test` and `--run`:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --perf-run <prun_...> --asserted "Asserted that \u2026" --org <org>\n   [cli >= 0.54.0]\n   For an eval run, use `--eval-run` in the same place:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_...> --asserted "Asserted that \u2026" --org <org>\n   [/cli]\n   An approved Workflow that is not a fork is attached the same way; the\n   result says `"existing": true`.\n   Attach exactly one evidence subject per command. A test containing any\n   `visual_checkpoint` node is a UI test, including a test that also has other\n   steps. Attaching it runs the same test at the PR base commit and the proof\n   page shows each checkpoint\'s base, head, and diff. When `--base` is omitted,\n   the CLI uses `git merge-base HEAD origin/<default>` when that ref is available.\n   Put checkpoints before\n   steps that depend on the new behavior so the base run can reach them.\n\n8. Push first, then render the criteria block:\n     cofactor proofs render --pr <n> --org <org>\n   The output already includes the markers. Paste the entire output into the\n   PR description, verbatim, replacing any existing proof block. Never\n   hand-format or edit it, and do not add a prose "Proof notes" section that\n   restates it. It looks like this:\n\n     <!-- cofactor:proof -->\n\n     **Proof of acceptance criteria** \xB7 [View all proofs in Cofactor](<proof page>)\n\n     \u{1F7E2} 1 passed \xB7 \u{1F7E1} 1 stale \xB7 \u26AA 1 manual \xB7 as of `a1b2c3d`\n\n     \u{1F7E2} **1. The saved filter stays selected after a reload** \xB7 Passed\n     - **Tested:** Filter bar keeps its selection \xB7 E2E run in a sandbox \xB7 [View the run](<run page>)\n       - Asserted that the Status filter still shows "Open" after a reload.\n\n     \u{1F7E1} **2. The list query is no slower** \xB7 Stale (proved at 9f8e7d6)\n     - **Tested:** Performance run (prod) \xB7 [See the comparison](<perf run page>)\n       - Asserted that the list query is no slower than base.\n\n     \u26AA **3. The desktop client shows the badge** \xB7 Manual\n     - **Tested:** Manual check, not provable with Cofactor \xB7 [See the proof](<proof page>)\n       - Opened the desktop build and checked the badge.\n\n   Each criterion leads with a stoplight: \u{1F7E2} passed, \u{1F7E1} passed at an earlier\n   commit (stale), \u{1F534} failed, \u26AA shown manually because Cofactor could not prove\n   it. Under it, each evidence item names the test (its user goal), how it\n   ran, what it asserted, and links to it. The block is a snapshot as of the\n   commit in its header: it goes stale after a push, and the live verdict is\n   the `cofactor / proofs` check and the proof page. After every push, attach,\n   or detach, render again and replace the block. Context for a reviewer goes\n   in the criterion\'s `--detail` and in the `Asserted that \u2026` line (the proof\n   page shows both), not in a paragraph in the PR body.\n\n9. Inspect or revise the proof:\n     cofactor proofs status --repo <owner/repo> --pr <n> --org <org>\n   The repo is inferred from `GITHUB_REPOSITORY` or `git remote get-url origin`\n   and can be overridden with `--repo`.\n   Detaching removes a criterion and all evidence attached to it:\n     cofactor proofs detach --pr <n> --criterion <n> --org <org>\n   Remove just one item with `--test <wf_...>`, `--perf-run <prun_...>` or\n   `--manual`:\n     cofactor proofs detach --pr <n> --criterion <n> --perf-run <prun_...> --org <org>\n   [cli >= 0.54.0]\n   `--eval-run <aerun_...>` removes one eval run the same way.\n   [/cli]\n   Render again and replace the PR proof block after every attach or detach.\n\nWHEN COFACTOR CANNOT PROVE A CRITERION\n\nManual evidence is for a criterion Cofactor cannot exercise at all (for example\nit needs a desktop client, real hardware or a third-party console). If the\ncriterion is plainly outside a browser or a sandbox, go straight to it;\notherwise try a Cofactor test first. A failing, flaky or hard-to-write test is\nnever a reason for manual evidence: fix the test or the change, and never detach\nfailing evidence to replace it with manual evidence.\n\nSay what you did and why Cofactor could not prove it, from a clean checkout at\nthe pushed commit:\n     cofactor proofs attach --pr <n> --application <app> --criterion <n> --text "<criterion>" --detail "<why this matters>" --manual --notes <text|@file> [--artifact <https-url>]... --org <org>\n   Notes (up to 10,000 characters) and links (up to 20) are shown to everyone\n   who can read the proof: keep notes to what you did and why, and redact\n   secrets, tokens and customer data from anything you link. Links must be\n   https URLs reviewers can open, never signed or tokenized links and never\n   with credentials (an image attached to the pull request works; where your\n   organization has Cofactor utilities, `cofactor utilities object upload\n   <file> --public --json` returns one).\n   A criterion holds one manual item, and attaching again replaces its notes\n   and links, so pass the same `--notes` and every `--artifact` again. A\n   criterion whose only evidence is manual passes the check with `\u{1F4DD}` and\n   `[manual]`, renders \u26AA Manual in the block, and shows a Manual badge with\n   your notes on the proof page; a criterion that mixes manual with other kinds\n   keeps its stoplight and is decided by its other items. Manual evidence never promotes\n   anything and reads stale after a push; attach it again at the new head.\n\nCHECKS AND STALE EVIDENCE\n\nThe App publishes `cofactor / proofs`; the Actions job reports the same server\nverdict. Diagnose with a body file:\n     cofactor proofs check --pr <n> --sha <head> --body-file <file> --org <org>\n   Or pass `--body -` for stdin. The App queues a refreshed check after attach\nor detach. In the Actions path, rerun the check job after attaching or\ndetaching. Successful stale evidence remains successful and is\nmarked stale. Use Re-run on the Cofactor proof page for stale E2E evidence at\nthe PR head. For stale performance evidence, run it at the new head, attach\nthat run and detach the old item. Cofactor does not rerun automatically. Server and CI own promotion\ntriggers; `cofactor trigger` is not your job.\n\nNo block means no proof check; a block with no criteria fails. When the pull\nrequest has no observable acceptance criterion at all (for example a\ndependency bump or a docs-only change), print the neutral block and paste it\nin place of any proof block:\n     cofactor proofs render --none "<why this pull request has no criterion to prove>"\nIt reports a neutral result. A criterion that only Cofactor cannot exercise\ngets manual evidence instead (above).\n\nCURATING THE SUITE\n\nEvery promoted test runs from then on, so keep the suite worth running:\n   - Reuse first: attach the run of an existing test or eval that already proves\n     the criterion before creating one.\n   - Extend an existing test instead of duplicating its journey with a new one.\n   - One test per user goal, named for that goal (what the user accomplishes,\n     not the component or the ticket).\n   - Promote lasting behaviour whose regression would matter.\n   - Set a Workflow to `--promote-on never` when its proof is about the change\n     itself: a one-time migration, backfill, redirect or flag rollout step,\n     behaviour about to be removed, or a one-off visual before/after of a\n     redesign.\n   - Perf and eval runs never join a suite through proofs: a perf A/B or an\n     eval experiment for one decision stays a run. Save a perf test only for a\n     hot path (see the perf guide).\n   - Don\'t keep a test nobody would miss if it disappeared.\n\nWHAT GOES IN THE PULL REQUEST\n\nPaste the output of `proofs render` and nothing that restates it: no "Proof\nnotes" section. If a reviewer needs to know why a Workflow was forked or what\nstays uncovered, say it once in the criterion\'s `--detail`, or make it a\ncriterion with manual evidence.\n\nUse neutral example names and never include customer names, domains, or\ncredentials. A human approves Workflows by merging the pull request; do not\napprove them yourself.\n';
 
 // ../content/src/proofs-guide/index.ts
 var PROOFS_GUIDE_SECTIONS = [
-  { title: "Prove a Change with Tests", slug: "overview", content: overview_default3 }
+  { title: "Prove a Change with Tests", slug: "overview", content: overview_default4 }
 ];
 
 // ../content/src/perf-guide/overview.txt
-var overview_default4 = "Use a perf test to compare two commits of the same Application in a sandbox:\nLighthouse for page loading, k6 for HTTP load, and server spans when the app\nexports OpenTelemetry (OTel). Choose a target metric and a goal before iterating.\n\nWRITE perf.yaml\n\n  rounds: 7\n  lighthouse:\n    formFactor: desktop\n    pages:\n      - { name: dashboard, path: /dashboard, as: admin }\n  k6:\n    script: perf/dashboard.k6.js\n    vus: 10\n    durationSeconds: 30\n    as: admin\n\n  - rounds: integer 5..15, default 7. Each arm (base or head) gets that many\n    measured rounds after a discarded warm-up: one Lighthouse load per page\n    and a 10-second k6 pass. Measured rounds alternate base/head then head/base;\n    each arm runs all Lighthouse pages before k6.\n  - Configure lighthouse, k6, or both. Omit a block to skip that flavor.\n  - lighthouse.formFactor: required desktop or mobile, with Lighthouse's stock\n    simulated throttling. pages: 1..10 entries, each with a required name and\n    path. name is 1..40 lowercase letters, digits or hyphens. path starts with\n    /, cannot start with //, and cannot contain backslashes.\n  - k6.script: required path relative to perf.yaml; the CLI inlines its contents.\n    vus: required integer 1..200. durationSeconds: required integer 10..300.\n  - as: optional persona name, per Lighthouse page or for k6. Omit for anonymous\n    access. Use a persona configured in the Application's sandbox definition.\n  - Configs whose estimated time for both arms exceeds 90 minutes are rejected;\n    reduce rounds, pages or k6 duration.\n\nWRITE THE k6 SCRIPT\n\n  Put one default function in perf/dashboard.k6.js:\n\n    import http from 'k6/http';\n\n    export default function () {\n      http.get(`${__ENV.BASE_URL}/api/projects`, {\n        headers: { Cookie: __ENV.COOKIE },\n      });\n    }\n\n  The runner supplies BASE_URL and COOKIE as environment variables, read through\n  __ENV. COOKIE contains the persona's cookie header (empty for anonymous).\n  Send it on your requests as above; setting as does not add the header to your\n  script. Load duration and VUs come from perf.yaml.\n\n  Lighthouse runs inside the sandbox against the app's PUBLIC port URL, with\n  the page persona's cookies installed in Chrome. It must finish on the requested\n  path with a 2xx document status; a login redirect fails the run. k6 uses that\n  same public sandbox base URL and sends the persona cookie through your script.\n\nENABLE SERVER SPANS (OPTIONAL)\n\n  Instrument your app with OTel and add these values to the sandbox definition's\n  top-level env: block:\n\n    env:\n      OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318\n      OTEL_SERVICE_NAME: acme\n      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf\n\n  Environment variables configure an instrumented app; they do not instrument it.\n  Apps without OTel lose nothing from Lighthouse or k6: leave the endpoint\n  undeclared and the result has otel: null and no otel rows. Declaring the endpoint\n  but receiving zero spans on either arm fails the run.\n\nRUN AND READ THE VERDICT\n\n    cofactor perf run --config perf.yaml --application acme \\\n      --environment preview --base <base-sha> --head <head-sha> --org acme\n\n  --head is required. --head and --base take what `cofactor sandbox claim --at`\n  takes: for an Application built from GitHub, a full 40-character lowercase\n  commit SHA; for an Application pushed as an image, a push id or a tag (for\n  example nightly). Omit --base to compare against the environment's\n  default-branch head (GitHub) or against the ready push before --head (pushed\n  image). Omit --environment to use the Application's default environment.\n  Both arms use the same Application and environment. The CLI waits for\n  completion.\n\n  The table shows base/head medians, delta, 95% CI and verdict. Relative rows\n  compare median(head) / median(base) - 1 with a 95% bootstrap confidence interval\n  (10,000 resamples). For a gated row, slower requires the entire CI above\n  +10% (minEffect) AND the absolute median difference beyond its floor; faster\n  requires the entire CI below -10% and the same floor. Otherwise: no_change.\n  Higher requests/s is better: JSON flips its delta sign so positive means worse;\n  the printed table shows throughput gains as positive.\n\n  Gated rows and absolute floors:\n    Lighthouse LCP: 100 ms; TBT: 50 ms, per page.\n    k6 p95 latency: 5 ms; requests/s: no extra floor; error rate: 1 percentage point.\n    OTel endpoint p95: 5 ms, only with data in every measured round on both arms\n      and at least 30 spans per arm for that endpoint.\n\n  Error-rate rows (unit ratio) and rows with any zero base-round value use an\n  absolute difference instead of a ratio. Their CI must clear zero and the\n  difference must exceed the floor; the 10% minEffect does not apply. Error-rate\n  deltas print as +x.x pt (percentage points), zero-base timing deltas as +x ms;\n  their CIs use the same units. JSON deltaPct and ci95 also hold those absolute\n  units for these rows, despite the field name.\n\n  FCP, CLS, speed index, bytes, p50s and insufficiently sampled OTel endpoints\n  are reported, never gated. Ungated rows always have verdict no_change.\n  Rule: never trust a single `faster` row of an ungated metric.\n\n  The run takes the worst gated verdict: any slower wins, otherwise any faster\n  wins, otherwise no_change. The table prints SLOWER for regressions; most\n  ungated rows are omitted (use --json for all rows). A slower run with OTel\n  includes slowest head spans and new or slower spans. Incomplete measurements,\n  failed Lighthouse navigation or any k6 round above 50% errors fail the run\n  instead of producing a verdict.\n\n  Exit codes: 0 = faster or no_change; 3 = slower; 1 = failed run, timeout or error.\n  Handle exit 3 in a shell loop. --json flattens the result into run metadata:\n  id, status, verdict, baseSha, headSha, rounds, rows[], otel and tools are at the\n  top level. Each row has gated, verdict, base/head median and values, deltaPct\n  and ci95. For example, count regressions after capturing a completed run:\n\n    cofactor perf run --config perf.yaml --application acme --head <head-sha> \\\n      --org acme --json > perf-result.json\n    jq '[.rows[] | select(.gated and .verdict==\"slower\")] | length' perf-result.json\n\nSAVE A DURABLE TEST\n\n    cofactor perf save --config perf.yaml --name dashboard --application acme \\\n      --environment preview --cron \"0 9 * * 1-5\" --timezone UTC --org acme\n    cofactor perf list --org acme\n    cofactor perf get dashboard --org acme\n    cofactor perf run --test dashboard --head <head-sha> --org acme\n    cofactor perf runs dashboard --limit 20 --org acme\n    cofactor perf get <prun_id> --json --org acme\n    cofactor perf delete dashboard --org acme\n\n  A scheduled run measures the default branch head against its first parent\n  commit (GitHub), or the default-tag push head against the previous ready push\n  (pushed image), and skips a head that already has a run; \"Run now\" is never\n  skipped.\n\n  Save a GitHub test with --on-merge to run it when a pull request or merged SHA\n  reaches the repository's default branch:\n\n    cofactor perf save --config perf.yaml --name dashboard \\\n      --application ExampleApp --on-merge --org ExampleOrg\n\n  Merge runs fire only when a merge reaches the default branch and compare the\n  merge commit with its first parent. A root commit with no first parent is\n  skipped. Repeated deliveries for a test and head SHA create at most one run.\n  Upload-delivered applications cannot use --on-merge. The trigger works\n  without a proof record; proofs promotion and perf measurement are independent.\n\n  A perf run attached as proof joins nothing at merge. Save a perf test (on a\n  schedule or --on-merge) only for a hot journey or endpoint: one ongoing perf\n  test each. An A/B for one change stays an ad hoc run.\n\n  save replaces the saved config, schedule, and on-merge marker by name. Omit\n  --on-merge to clear that marker. Without timing flags and --on-merge, the test\n  is manual. For a schedule, supply exactly one timing flag: --schedule \"every weekday at 9am\",\n  --cron \"0 9 * * 1-5\", --at 09:00, or --every 2h. --timezone defaults to UTC.\n  --test uses the saved Application, environment and inlined script; do not pass\n  --application or --environment with it. get accepts a test name, ptest_ id or\n  prun_ id. runs lists newest first (--limit 1..100, default 20). delete removes\n  the saved test, its automation rules and its runs.\n\n  Open the Performance page (/performance) to browse saved tests and recent\n  ad hoc runs, inspect a test's history and metric confidence intervals, or open\n  a run's verdict table, tool versions and server spans for both arms.\n\nTHE hill-climb LOOP\n\n  0. Every new SHA or push needs a snapshot and a perf layer before its sandboxes are\n     ready. Builds take minutes (up to 90), so batch changes per iteration.\n  1. Run once on the unchanged branch. If the verdict surprises you, do an A/A\n     sanity check: set --base and --head to the same full SHA.\n  2. Read the worst gated row. If otel is present, inspect its endpoint rows and\n     slow spans (otel.topSlowSpans.base/head in --json).\n  3. Change code, commit and push.\n  4. Measure the new commit, keeping the same base for the comparison:\n       cofactor perf run --config perf.yaml --application acme \\\n         --base <base-sha> --head <new-sha> --org acme\n  5. Keep the commit if faster, or no_change with a better target median. Revert\n     it if slower. Lower latency is better; higher requests/s is better.\n  6. Stop when no gated row is slower and the target row is under its goal.\n\nREFERENCE\n\n  - Current flags: `cofactor perf --help`\n  - Sandbox definitions and personas: `cofactor sandbox guide v2`\n";
+var overview_default5 = "Use a perf test to compare two commits of the same Application in a sandbox:\nLighthouse for page loading, k6 for HTTP load, and server spans when the app\nexports OpenTelemetry (OTel). Choose a target metric and a goal before iterating.\n\nWRITE perf.yaml\n\n  rounds: 7\n  lighthouse:\n    formFactor: desktop\n    pages:\n      - { name: dashboard, path: /dashboard, as: admin }\n  k6:\n    script: perf/dashboard.k6.js\n    vus: 10\n    durationSeconds: 30\n    as: admin\n\n  - rounds: integer 5..15, default 7. Each arm (base or head) gets that many\n    measured rounds after a discarded warm-up: one Lighthouse load per page\n    and a 10-second k6 pass. Measured rounds alternate base/head then head/base;\n    each arm runs all Lighthouse pages before k6.\n  - Configure lighthouse, k6, or both. Omit a block to skip that flavor.\n  - lighthouse.formFactor: required desktop or mobile, with Lighthouse's stock\n    simulated throttling. pages: 1..10 entries, each with a required name and\n    path. name is 1..40 lowercase letters, digits or hyphens. path starts with\n    /, cannot start with //, and cannot contain backslashes.\n  - k6.script: required path relative to perf.yaml; the CLI inlines its contents.\n    vus: required integer 1..200. durationSeconds: required integer 10..300.\n  - as: optional persona name, per Lighthouse page or for k6. Omit for anonymous\n    access. Use a persona configured in the Application's sandbox definition.\n  - Configs whose estimated time for both arms exceeds 90 minutes are rejected;\n    reduce rounds, pages or k6 duration.\n\nWRITE THE k6 SCRIPT\n\n  Put one default function in perf/dashboard.k6.js:\n\n    import http from 'k6/http';\n\n    export default function () {\n      http.get(`${__ENV.BASE_URL}/api/projects`, {\n        headers: { Cookie: __ENV.COOKIE },\n      });\n    }\n\n  The runner supplies BASE_URL and COOKIE as environment variables, read through\n  __ENV. COOKIE contains the persona's cookie header (empty for anonymous).\n  Send it on your requests as above; setting as does not add the header to your\n  script. Load duration and VUs come from perf.yaml.\n\n  Lighthouse runs inside the sandbox against the app's PUBLIC port URL, with\n  the page persona's cookies installed in Chrome. It must finish on the requested\n  path with a 2xx document status; a login redirect fails the run. k6 uses that\n  same public sandbox base URL and sends the persona cookie through your script.\n\nENABLE SERVER SPANS (OPTIONAL)\n\n  Instrument your app with OTel and add these values to the sandbox definition's\n  top-level env: block:\n\n    env:\n      OTEL_EXPORTER_OTLP_ENDPOINT: http://127.0.0.1:4318\n      OTEL_SERVICE_NAME: acme\n      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf\n\n  Environment variables configure an instrumented app; they do not instrument it.\n  Apps without OTel lose nothing from Lighthouse or k6: leave the endpoint\n  undeclared and the result has otel: null and no otel rows. Declaring the endpoint\n  but receiving zero spans on either arm fails the run.\n\nRUN AND READ THE VERDICT\n\n    cofactor perf run --config perf.yaml --application acme \\\n      --environment preview --base <base-sha> --head <head-sha> --org acme\n\n  --head is required. --head and --base take what `cofactor sandbox claim --at`\n  takes: for an Application built from GitHub, a full 40-character lowercase\n  commit SHA; for an Application pushed as an image, a push id or a tag (for\n  example nightly). Omit --base to compare against the environment's\n  default-branch head (GitHub) or against the ready push before --head (pushed\n  image). Omit --environment to use the Application's default environment.\n  Both arms use the same Application and environment. The CLI waits for\n  completion.\n\n  The table shows base/head medians, delta, 95% CI and verdict. Relative rows\n  compare median(head) / median(base) - 1 with a 95% bootstrap confidence interval\n  (10,000 resamples). For a gated row, slower requires the entire CI above\n  +10% (minEffect) AND the absolute median difference beyond its floor; faster\n  requires the entire CI below -10% and the same floor. Otherwise: no_change.\n  Higher requests/s is better: JSON flips its delta sign so positive means worse;\n  the printed table shows throughput gains as positive.\n\n  Gated rows and absolute floors:\n    Lighthouse LCP: 100 ms; TBT: 50 ms, per page.\n    k6 p95 latency: 5 ms; requests/s: no extra floor; error rate: 1 percentage point.\n    OTel endpoint p95: 5 ms, only with data in every measured round on both arms\n      and at least 30 spans per arm for that endpoint.\n\n  Error-rate rows (unit ratio) and rows with any zero base-round value use an\n  absolute difference instead of a ratio. Their CI must clear zero and the\n  difference must exceed the floor; the 10% minEffect does not apply. Error-rate\n  deltas print as +x.x pt (percentage points), zero-base timing deltas as +x ms;\n  their CIs use the same units. JSON deltaPct and ci95 also hold those absolute\n  units for these rows, despite the field name.\n\n  FCP, CLS, speed index, bytes, p50s and insufficiently sampled OTel endpoints\n  are reported, never gated. Ungated rows always have verdict no_change.\n  Rule: never trust a single `faster` row of an ungated metric.\n\n  The run takes the worst gated verdict: any slower wins, otherwise any faster\n  wins, otherwise no_change. The table prints SLOWER for regressions; most\n  ungated rows are omitted (use --json for all rows). A slower run with OTel\n  includes slowest head spans and new or slower spans. Incomplete measurements,\n  failed Lighthouse navigation or any k6 round above 50% errors fail the run\n  instead of producing a verdict.\n\n  Exit codes: 0 = faster or no_change; 3 = slower; 1 = failed run, timeout or error.\n  Handle exit 3 in a shell loop. --json flattens the result into run metadata:\n  id, status, verdict, baseSha, headSha, rounds, rows[], otel and tools are at the\n  top level. Each row has gated, verdict, base/head median and values, deltaPct\n  and ci95. For example, count regressions after capturing a completed run:\n\n    cofactor perf run --config perf.yaml --application acme --head <head-sha> \\\n      --org acme --json > perf-result.json\n    jq '[.rows[] | select(.gated and .verdict==\"slower\")] | length' perf-result.json\n\nSAVE A DURABLE TEST\n\n    cofactor perf save --config perf.yaml --name dashboard --application acme \\\n      --environment preview --cron \"0 9 * * 1-5\" --timezone UTC --org acme\n    cofactor perf list --org acme\n    cofactor perf get dashboard --org acme\n    cofactor perf run --test dashboard --head <head-sha> --org acme\n    cofactor perf runs dashboard --limit 20 --org acme\n    cofactor perf get <prun_id> --json --org acme\n    cofactor perf delete dashboard --org acme\n\n  A scheduled run measures the default branch head against its first parent\n  commit (GitHub), or the default-tag push head against the previous ready push\n  (pushed image), and skips a head that already has a run; \"Run now\" is never\n  skipped.\n\n  Save a GitHub test with --on-merge to run it when a pull request or merged SHA\n  reaches the repository's default branch:\n\n    cofactor perf save --config perf.yaml --name dashboard \\\n      --application ExampleApp --on-merge --org ExampleOrg\n\n  Merge runs fire only when a merge reaches the default branch and compare the\n  merge commit with its first parent. A root commit with no first parent is\n  skipped. Repeated deliveries for a test and head SHA create at most one run.\n  Upload-delivered applications cannot use --on-merge. The trigger works\n  without a proof record; proofs promotion and perf measurement are independent.\n\n  A perf run attached as proof joins nothing at merge. Save a perf test (on a\n  schedule or --on-merge) only for a hot journey or endpoint: one ongoing perf\n  test each. An A/B for one change stays an ad hoc run.\n\n  save replaces the saved config, schedule, and on-merge marker by name. Omit\n  --on-merge to clear that marker. Without timing flags and --on-merge, the test\n  is manual. For a schedule, supply exactly one timing flag: --schedule \"every weekday at 9am\",\n  --cron \"0 9 * * 1-5\", --at 09:00, or --every 2h. --timezone defaults to UTC.\n  --test uses the saved Application, environment and inlined script; do not pass\n  --application or --environment with it. get accepts a test name, ptest_ id or\n  prun_ id. runs lists newest first (--limit 1..100, default 20). delete removes\n  the saved test, its automation rules and its runs.\n\n  Open the Performance page (/performance) to browse saved tests and recent\n  ad hoc runs, inspect a test's history and metric confidence intervals, or open\n  a run's verdict table, tool versions and server spans for both arms.\n\nTHE hill-climb LOOP\n\n  0. Every new SHA or push needs a snapshot and a perf layer before its sandboxes are\n     ready. Builds take minutes (up to 90), so batch changes per iteration.\n  1. Run once on the unchanged branch. If the verdict surprises you, do an A/A\n     sanity check: set --base and --head to the same full SHA.\n  2. Read the worst gated row. If otel is present, inspect its endpoint rows and\n     slow spans (otel.topSlowSpans.base/head in --json).\n  3. Change code, commit and push.\n  4. Measure the new commit, keeping the same base for the comparison:\n       cofactor perf run --config perf.yaml --application acme \\\n         --base <base-sha> --head <new-sha> --org acme\n  5. Keep the commit if faster, or no_change with a better target median. Revert\n     it if slower. Lower latency is better; higher requests/s is better.\n  6. Stop when no gated row is slower and the target row is under its goal.\n\nREFERENCE\n\n  - Current flags: `cofactor perf --help`\n  - Sandbox definitions and personas: `cofactor sandbox guide v2`\n";
 
 // ../content/src/perf-guide/index.ts
 var PERF_GUIDE_SECTIONS = [
-  { title: "Performance Tests", slug: "overview", content: overview_default4 }
+  { title: "Performance Tests", slug: "overview", content: overview_default5 }
 ];
 
 // ../content/src/eval-guide/overview.txt
-var overview_default5 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 3                      # attempts per case, 1 to 7\n  passAtLeast: 2                  # a case passes when this many attempts pass\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n';
+var overview_default6 = 'Use an eval to measure an AI agent your product ships: run its command against a\nset of cases at a commit, grade every attempt, and get a pass rate, cost and\ntime per task. Run, grade and compare models with the CLI; no GitHub setup is\nneeded beyond the commit you evaluate. Pass `--org <org>` on every write.\n\nWHEN TO WRITE AN EVAL\n\n- An agent\'s behavior is the thing that can regress: a prompt, a tool set, a\n  model change, a retrieval change.\n- A workflow test cannot decide it, because the output is free text or a\n  judgment (a diagnosis, a summary, a support reply), not a click path.\n- You can name real cases and what a good answer looks like. Without cases,\n  write a workflow test (`cofactor workflow guide write-a-test`) instead.\n- Do not eval deterministic code; unit-test it.\n\nSTART\n\n  cofactor eval init <name> --application <app> --capability CAP-XXX-001\n\nThis writes evals/<name>.eval.yaml, evals/<name>.runbook.md and\nevals/.gitignore, and refuses to overwrite. Edit them, commit them with the\nagent code, then push:\n\n  cofactor eval push evals/<name>.eval.yaml --application <app> --org <org>\n\nPush validates the template locally, stores it with its runbook and is\nidempotent. `cofactor eval show <name>` reads it back.\n\nTHE TEMPLATE\n\n  version: 1\n  name: diag-smoke                # 2 to 40 lowercase letters, digits, dashes\n  description: One sentence on what the agent must get right.\n  proves:\n    capabilities: [CAP-XXX-001]   # product-map codes in the pushed application\n  runbook: diag-smoke.runbook.md  # path relative to the YAML\n  mode: single-turn               # or multi-turn (needs simulator)\n  run: ./scripts/run-agent.sh     # shell command at the repo root of the commit\n  input:\n    description: What the files in a case\'s input directory contain.\n    require: [input.json]         # every case bundle must contain these paths\n  agent:\n    models:\n      sonnet: { provider: anthropic, model: claude-sonnet-4-6 }\n    default: sonnet               # aliases are what --model takes\n    providers:                    # optional: only when the agent reads non-standard\n      azure:                      # names (see CONFIGURE THE AGENT\'S MODEL below)\n        env: { apiKey: LLM_API_KEY, baseUrl: LLM_BASE_URL, model: LLM_MODEL }\n  graders:                        # at least one\n    - { id: answer-matches, kind: script, command: ./scripts/grade.sh }\n    - id: quality\n      kind: ai-judge\n      rubric: "At least 20 characters: what each score means."\n      scale: 1-5                  # or binary\n      passAt: 4                   # required for 1-5, 2 to 5\n      judge: { provider: anthropic, model: claude-opus-4-8 }\n  repeats: 3                      # attempts per case, 1 to 7\n  passAtLeast: 2                  # a case passes when this many attempts pass\n  passBar: 0.5                    # share of cases that must pass, 0.1 to 1\n  attemptSeconds: 600             # per attempt, 30 to 1700\n\nUnknown fields are rejected. Set passBar from a measured baseline and treat it\nas a regression ratchet, not an aspiration.\n\nTHE STANDARD ENVIRONMENT\n\nEvery attempt runs `run` in the repository root of the evaluated commit, in an\nisolated sandbox, with:\n\n  EVAL_INPUT          the case\'s input directory (read only what you need)\n  EVAL_OUTPUT         the directory to write the answer to\n  EVAL_MODEL          the model id for this run\n  EVAL_PROVIDER       anthropic or azure\n  EVAL_CASE           the case key\n  EVAL_ATTEMPT        the attempt index\n  EVAL_RUN_ID         the run id\n  EVAL_ATTEMPT_SECONDS  the time limit\n\nplus, for the selected model\'s provider only, a key, a base URL that routes\nmodel calls through Cofactor (which meters cost and latency) and the model id.\nBy default they use the standard names:\n\n  anthropic:  ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL,\n              ANTHROPIC_MODEL\n  azure:      OPENAI_API_KEY, AZURE_OPENAI_API_KEY, OPENAI_BASE_URL,\n              AZURE_OPENAI_BASE_URL, OPENAI_MODEL\n\nThe key holds a short-lived token for that proxy, never a real provider key.\nThe base URL speaks the provider\'s own wire: the Anthropic Messages API for\nanthropic, the OpenAI v1 API (chat completions and responses) for azure.\nEvery other provider-key secret your sandbox declares (the other provider\'s,\nGemini\'s and Google\'s, and with an agent.providers entry also the selected\nprovider\'s standard names) arrives empty (set, not absent), so a call to a\nprovider you did not select fails. Plain `env:` values in your sandbox\ndefinition (not secrets) reach the attempt unchanged. Do not\nhard-code a key, a URL or a model id; read them from the environment. The case\'s\nexpected value is withheld from the agent and only reaches the graders.\n\nCONFIGURE THE AGENT\'S MODEL\n\nMost agents need nothing here. Add an `agent.providers` entry only for a\nprovider whose agent reads names other than the standard ones (an entry for a\nprovider no model uses is an error). An entry replaces all of that provider\'s\nstandard names: the attempt then sets exactly these three, and nothing else.\n\n  agent:\n    providers:\n      azure:\n        env:\n          apiKey: LLM_API_KEY      # gets the proxy token\n          baseUrl: LLM_BASE_URL    # gets the proxy URL\n          model: LLM_MODEL         # gets the model id\n\nNames are upper case, distinct, and never EVAL_*. A mapped key name that your\nsandbox also declares as a secret is replaced for the whole attempt (with the\ntoken, or emptied when another provider runs), so never map onto a secret the\napplication itself needs. Fill the entry from the\nagent\'s own config code, not by guessing: find where it builds its model client\nand copy the names it reads. For example, if the agent has\n\n  createAzure({\n    apiKey: process.env.LLM_API_KEY,\n    baseURL: process.env.LLM_BASE_URL,\n  })\n  const model = process.env.LLM_MODEL\n\nthen the entry above is right. Map baseUrl to the name the client really reads:\na client that ignores it calls the vendor directly with the proxy token, and\nthe call fails. If the client takes no base URL setting, add one first.\n\nWRITE THE ENTRYPOINT\n\nRead the case from `$EVAL_INPUT`, call the agent exactly as production does,\nand write the final answer to `$EVAL_OUTPUT/answer.<ext>` (answer.json for\nstructured output, answer.txt for prose). The first `answer.*` file in sorted\norder is graded (so write exactly one), up to 256 KiB. No answer file is a\nfailed attempt. Anything the script prints goes to stdout.log in the output\ndirectory, which helps when debugging; do not print secrets or dump the\nenvironment.\n\n  #!/usr/bin/env bash\n  set -euo pipefail\n  bun run scripts/my-agent.ts \\\n    --input "$EVAL_INPUT/input.json" --out "$EVAL_OUTPUT/answer.json"\n\nRun the real code path, not a mock of it. Exit non-zero when the agent breaks\n(it is recorded for debugging; the answer file decides the grade).\n\nCHOOSE GRADERS\n\nScript first. Use a judge only for what code cannot decide. A case passes an\nattempt only when every grader passes.\n\n- Script grader: runs after the agent exits, with EVAL_INPUT, EVAL_OUTPUT and\n  EVAL_EXPECTED (the case\'s expected value as a JSON file), in the repo root,\n  with a fixed 120 second limit. Its last stdout line must be\n  {"pass": true, "score": 1, "reason": "why"}. Anything else is a grader\n  error, which is reported as an error, never as a pass or a fail. Use scripts\n  for categories, ids, exact fields, schema checks and state.\n- ai-judge grader: scores a rubric against the answer and the expected value.\n  Use binary for a fact ("names the same cause: yes or no") and 1 to 5 for a\n  quality (completeness, tone), with the meaning of each score in the rubric\n  and `passAt` set. Always pin the judge model; never leave it to a default.\n  Judge the content, not the wording.\n\nBUILD CASES\n\nEach case is an input directory, an expected JSON value, and a provenance.\n\n  cofactor eval add-case <name> --key <k> --input <dir|file.tar.gz> \\\n    --expected <file|-> --source "<provenance>" [--title <t>] --org <org>\n\n- Read the runbook first: `cofactor eval runbook <name>` works with no checkout.\n- One idea per case. A case that tests three things hides which one broke.\n- Prefer real inputs. Sanitize them before upload: no customer names, domains,\n  people, tokens or ids. Use neutral names (Acme, example.com). Customer data\n  stays in the database, never in a case.\n- Label from something other than the agent\'s own output (a human decision, a\n  fix that landed, a recorded outcome). Say what corroborates it in `--source`.\n- When no real case exists, mutate a frozen real input and mark it synthetic in\n  `--source`.\n- Add every failing case you find, as you find it. The set should grow from\n  real misses.\n- `--input` takes a directory (tarred deterministically) or a .tar.gz; the\n  template\'s `input.require` paths are checked first. `--expected -` reads\n  stdin. A bundle already stored is deduplicated.\n- `cofactor eval cases <name>` lists them; `cofactor eval rm-case <name> <key>`\n  removes one no run has used.\n\nMULTI-TURN\n\nSet `mode: multi-turn` and add `simulator: { provider, model }` (pinned). `run`\nis then the agent process, not a one-shot command. It speaks JSONL on stdin and\nstdout: read one line per user message, answer with one line.\n\n  stdin   {"user": "message"}\n  stdout  {"reply": "answer"}\n  either side may end it with {"done": true}\n\nNo answer file is needed in multi-turn mode; the graded input is the\nconversation. stdout belongs to the protocol, so any other line you print on\nstdout (a debug message, a log) is a protocol error that errors the attempt;\nlog to stderr. An agent that exits before replying, or writes an over-long line,\nis also a protocol error. As in single-turn mode, a multi-turn attempt that runs\nout of time is not an error: it is graded on the turns completed so far, and\ncounts as a fail if none completed. Multi-turn output also lands in\nstdout.log. A protocol error is not retried.\n\nEach case needs a conversation: `--conversation <file.json>`, either\n`{"kind":"scripted","turns":["first message", "second message"]}` or\n`{"kind":"simulated","persona":"\u2026","goal":"\u2026","facts":["\u2026"],"stopWhen":"\u2026",\n"maxTurns":8}`. A simulated user is another model playing the persona, and the\ngoal and facts never reach the agent. Prefer a script grader that inspects the\nresulting state; use a judge over the transcript for process rules. Read a\ntranscript with `cofactor eval get <run-id> --case <key> --transcript`.\n\nRUN\n\n  cofactor eval run <name> [--model <alias>]... [--environment <env>]\n                    [--at <sha>] [--case <key>]... [--repeats N]\n                    [--parallel N] [--no-wait] --org <org>\n\nOne run per `--model` (none means the default). Attempts run `--parallel` at a\ntime; raise it before cutting cases when a large set times out. Cost and time\ngrow with cases x repeats x models. It waits, then prints a\nsection per model and a comparison grid. `--at <sha>` evaluates a commit; push\nthe commit first. `--case` (a subset of cases) and a `--repeats` that differs\nfrom the template make a partial run, useful for iterating, which never counts\nas coverage or proof. Ids are\nprinted to stderr first; `--no-wait` prints them and exits 0.\n\nEXIT CODES (run)\n\n  0  every run\'s verdict is pass\n  3  at least one run\'s verdict is fail (the agent missed the pass bar)\n  1  a failed or cancelled run, a polling timeout, or any error\nExit 1 wins over 3: if any run failed or was cancelled, the code is 1 even when\nanother run\'s verdict is fail. A polling timeout leaves the runs going; read\nthem later with `cofactor eval get`. All other sub-commands exit 0 on success\nand 1 on error. Gate on 0.\n\nREAD RESULTS\n\n  cofactor eval get <run-id>                 summary, cases, grader reasons\n  cofactor eval get <run-id> --case <key>    one case, every attempt\n  cofactor eval runs <name> [--limit N]      recent runs\n  cofactor eval list                         evals and their latest verdicts\n\nA case passes when `passAtLeast` of its `repeats` attempts pass; the eval\npasses when the share of passing cases reaches `passBar`. Read the grader\n`reason` before changing anything.\n\nA single-turn attempt is graded on its answer file. The agent command\'s exit code\nand a timeout are recorded but do not by themselves fail the attempt: a crash\nor a timeout that left no `answer.*` is a counted fail ("no answer written"),\nand one that left a good answer can still pass. An attempt is errored, not\nfailed, when a grader failed (a script grader printed no parsable last line or\nran past its limit, since its own exit code is ignored and only its last line\ncounts; or an ai-judge call failed), the answer file is over 256 KiB, a\nmulti-turn protocol error occurred, or the sandbox failed (infrastructure\nfailures are retried once). One errored attempt makes the whole run `failed`\nwith no verdict, however many cases passed, and `run` exits 1, not 3. Fix the\ngrader, answer size or infrastructure problem and run again; the error text\nnames the case and attempt. The summary counts errored attempts as\n`infraFailures`.\n\nCOMPARE MODELS\n\n  cofactor eval run <name> --model sonnet --model haiku --org <org>\n  cofactor eval compare <run-id> <run-id>\n  cofactor eval compare --batch <id>\n\nOne grid, one row per case, one column per model, with pass rate, cost per task\nand time per task. Compare on the same commit and the same cases; the output\nshows the commit and a digest of the case set. Only a full run of the\ntemplate\'s default model alias counts as coverage; runs of other aliases are\nfor comparing.\n\nLINK WHAT IT PROVES\n\n- Capabilities: `proves.capabilities` in the template links them when you push.\n  Add or remove links later with\n  `cofactor product-map capability link-eval <CODE|id> --eval <name> --application <app> --org <org>`\n  (and `unlink-eval`), but `eval push` makes `proves.capabilities` the source\n  of truth: the next push unlinks any capability not listed there. Edit the\n  template, not the link.\n- Rules: `cofactor product-map rule link-eval <prule_\u2026> --eval <name> --org <org>`\n  ties an eval to a rule in a capability (and `unlink-eval`). The rule must be\n  active, its capability must sit in an area, and that capability and area must\n  belong to the eval\'s application. Pushing does not touch rule links.\n- An agent as a product-map area, for what an agent can do and evals prove:\n  `cofactor product-map area create <title> --kind agent --application <app> --org <org>`.\n- A pull request: attach a full run at the PR head commit with\n  `cofactor proofs attach --pr <n> --application <app> --criterion <n> --eval-run <aerun_\u2026> --asserted "Asserted that \u2026" --org <org>`.\n  The run must be completed, at the commit you pushed, full (not partial) and\n  passing its pass bar. The default-alias rule above is about product-map\n  coverage only; it does not apply to attachment: no model is privileged, and a\n  full passing run of any model alias proves the criterion (each item shows its\n  model). Attach as many runs as you have; every one must pass. Read\n  `cofactor proofs guide` for the whole procedure.\n\nTHE HILL-CLIMB LOOP\n\nThe hill-climb loop uses `--json` to climb on measured cases, not on feel.\n\n1. Run the eval at the current commit and note the pass rate.\n     cofactor eval run <name> --json --org <org> > run.json\n2. List the failing cases (run --json is an array, one entry per --model;\n   .[0] is the first model. A run whose status is failed has a null verdict:\n   read .[0].error, and the attempts in .[0].cases[].attempts[] whose status is\n   error. A run that never produced a result prints only id, status, verdict and\n   error, with no `cases`):\n     jq \'[.[0].cases[] | select(.passed|not) | .key]\' run.json\n   For a run id, `cofactor eval get <run-id> --json` returns the result\n   directly: jq \'[.cases[] | select(.passed|not) | .key]\'\n3. Read one case: `cofactor eval get <run-id> --case <key>`.\n4. Make one change to the agent that explains the failures, commit and push it.\n5. Iterate on the failing cases only (`--case <key>`), then run the full set at\n   the new head. Keep the change only if the full pass rate goes up and cost or\n   time did not regress. When it does not, revert and try another idea.\n\nNever tune `passBar`, a rubric or a grader to turn a failing run green; fix the\nagent, or fix a wrong label and say why in the commit. Never trust a pass\nfrom a partial run. Add a case for each new failure you meet.\n\nTHE RUNBOOK FORMAT\n\nEvery eval has a runbook at evals/<name>.runbook.md, stored with the eval and\nprinted by `cofactor eval runbook <name>`. Its headings are fixed so an agent\ncan rely on them. `cofactor eval init` writes all of them, each starting with a\n`<!-- cofactor:todo -->` line and a hint:\n\n  # <eval name> runbook\n  ## What this eval proves        the agent\'s job in one paragraph and the capabilities it backs\n  ## Anatomy of a case            the files in $EVAL_INPUT, what expected holds, what a good case teaches\n  ## Finding real data            where in prod the inputs live: exact cofactor commands, ids, how to pick varied cases\n  ## Labeling                     how expected is decided and what must corroborate it; never the agent\'s own output alone\n  ## Sanitizing                   what to strip or replace before upload; the check to run; neutral names to use\n  ## Synthesizing scenarios       how to make a case when no real one exists, and how to mark it synthetic in --source\n  ## Adding the case              the exact eval add-case command shape for this eval\n  ## Known traps                  what has gone wrong before\n\nWrite each section, then delete its marker line. `eval push` refuses a runbook\nwhere any section still has the marker, warns when a heading is missing, and\nerrors when "Finding real data" or "Sanitizing" is empty. Keep the runbook true:\nadd a trap when a case goes wrong.\n\nSCOPES AND ACCESS\n\nThe sandbox-v2 organization flag must be on. API keys need explicit scopes:\n\n  run:                                  sandbox:create and sandbox:exec, named\n                                        explicitly (sandbox:* does not grant exec)\n  push, add-case, rm-case:              sandbox:edit\n  show, list, cases, runbook, get,\n  runs, compare:                        sandbox:read\n\n`cofactor eval --help` lists every flag. `cofactor eval guide` needs no login.\n';
 
 // ../content/src/eval-guide/index.ts
 var EVAL_GUIDE_SECTIONS = [
-  { title: "Agent Evals", slug: "overview", content: overview_default5 }
+  { title: "Agent Evals", slug: "overview", content: overview_default6 }
 ];
 
 // ../content/src/session-guide/start.txt
-var start_default = 'Drive a real browser from the CLI to explore or verify a web app. You work\nthrough snapshots (an accessibility tree plus a screenshot) and issue commands\nagainst element refs.\n\nSTART A SESSION\n\n  cofactor session start --url https://app.example.com\n  cofactor session start --credential                       (interactive picker)\n  cofactor session start --credential "Admin User" --url https://app.example.com\n  cofactor session start --storage-state ./auth.json --url <url>\n  cofactor session start --headless --url <url>\n\n  Credentials belong to an application and carry saved login state, so a\n  session started with --credential is already signed in. Load one into a\n  running session with `cofactor session load-credential`.\n\n  Credentials only work for the deployed app they were saved against. Their\n  cookies are bound to that domain. For localhost or 127.0.0.1, start without\n  --credential, ask the user to sign in in the browser window, and wait for\n  them to confirm before continuing (or pass --storage-state with a local file).\n\nMANAGE SESSIONS\n\n  cofactor session list\n  cofactor session status\n  cofactor session stop            (or: stop --all)\n  cofactor session snapshot --session s1     (target one when several exist)\n\n  Always stop sessions you started when you finish.\n\nTHE LOOP\n\n  snapshot -> read -> act -> snapshot -> verify\n  Never assume an action worked; look at the page afterwards.\n  Run `cofactor session --help` for the full command list.\n';
+var start_default = 'Drive a real browser from the CLI to explore or verify a web app. You work\nthrough snapshots (an accessibility tree plus a screenshot) and issue commands\nagainst element refs.\n\nSTART A SESSION\n\n  cofactor session start --url https://app.example.com\n  cofactor session start --credential                       (interactive picker)\n  cofactor session start --credential "Admin User" --url https://app.example.com\n  cofactor session start --storage-state ./auth.json --url <url>\n  cofactor session start --headless --url <url>\n\n  Credentials belong to an application and carry saved login state, so a\n  session started with --credential is already signed in. Load one into a\n  running session with `cofactor session load-credential`.\n\n  Credentials only work for the deployed app they were saved against. Their\n  cookies are bound to that domain. For localhost or 127.0.0.1, start without\n  --credential, ask the user to sign in in the browser window, and wait for\n  them to confirm before continuing (or pass --storage-state with a local file).\n\nMANAGE SESSIONS\n\n  cofactor session list\n  cofactor session status\n  cofactor session stop            (or: stop --all)\n  cofactor session snapshot --session s1     (target one when several exist)\n\n  Always stop sessions you started when you finish.\n\nTHE LOOP\n\n  snapshot -> read -> act -> snapshot -> verify\n  Never assume an action worked; look at the page afterwards.\n  Run `cofactor session --help` for the full command list.\n\nHAND THE EXPLORING TO AN AGENT\n\n[cli >= 0.55.0]\n  To have a Cofactor agent explore an application from a prompt and report\n  what it finds, instead of driving the browser yourself, read\n  `cofactor exploratory-test guide`.\n[/cli]\n\nSAFETY\n\n  Sign in with a `$CREDENTIAL` placeholder or `cofactor session load-credential`\n  (see the `sign-in` section). Never print or type a real password or token into\n  the page or the transcript.\n';
 
 // ../content/src/session-guide/interact.txt
 var interact_default = 'Look at the page, act on it by ref, then check the result.\n\nSNAPSHOT\n\n  cofactor session snapshot                       tree + screenshot\n  cofactor session snapshot --search "Submit"     find elements fast\n  cofactor session snapshot --search "Email|Password"\n  cofactor session snapshot --expand e50          open a collapsed section\n  cofactor session snapshot --mode tree           tree only\n  cofactor session snapshot --mode screenshot     image only\n\n  The tree lists elements with refs like [ref=e15]. Use those refs in every\n  command below. --search is case-insensitive and OR-matched; use it to locate\n  elements, not to judge visual state (the highlight can hide subtle changes).\n  Collapsed regions show "(127 elements - use expand=e50 to see content)".\n\nACT\n\n  cofactor session click --ref e15 --element "Submit button"\n  cofactor session type --ref e7 --text "hello" --element "Search box" --submit\n  cofactor session fill --fields \'[{"ref":"e3","value":"John"},{"ref":"e4","value":"j@example.com"}]\'\n  cofactor session select --ref e12 --value admin --element "Role dropdown"\n  cofactor session press --key Enter\n  cofactor session hover --ref e8 --element "User menu"\n  cofactor session scroll --direction down --amount 5\n  cofactor session click --x 450 --y 300 --element "Canvas"     (last resort)\n\nNAVIGATE AND WAIT\n\n  cofactor session navigate --url https://app.example.com/settings\n  cofactor session back\n  cofactor session wait --text "Saved"             wait for text to appear\n  cofactor session wait --text "Loading..."        wait for text to disappear\n  cofactor session tabs --action list|new|select|close [--index n]\n\n  Prefer waiting on expected text over fixed delays.\n\nCAPTURE AND INSPECT\n\n  cofactor session screenshot --label before-submit [--full-page]\n  cofactor session console --only-errors\n  cofactor session network\n  cofactor session evaluate --js "() => document.title"\n  cofactor session downloads / read-download\n\n  `evaluate` reads page state only. Use `run-code` (driver-side Playwright)\n  when you need real click and keyboard events.\n\nVERIFY\n\n  After each action: snapshot, search for the success or error message\n  ("Success|saved", "Error|failed|invalid"), check console and network for\n  failed requests, and report exactly what you saw, including refs and error\n  text.\n';
@@ -1938,6 +1946,7 @@ function installConsoleBranding() {
 export {
   SANDBOX_GUIDE_SECTIONS,
   EXTERNAL_RUN_GUIDE_SECTIONS,
+  EXPLORATORY_TEST_GUIDE_SECTIONS,
   ISSUES_GUIDE_SECTIONS,
   PROOFS_GUIDE_SECTIONS,
   PERF_GUIDE_SECTIONS,
@@ -1961,4 +1970,4 @@ export {
   shouldBrandConsoleOutput,
   installConsoleBranding
 };
-//# sourceMappingURL=chunk-KNWKOKM7.js.map
+//# sourceMappingURL=chunk-KL2ARGH6.js.map