code-foundry 1.22.1 → 1.28.2

package/.github/workflows/validation-no-codeql.yml

@@ -42,6 +42,14 @@ on:
         required: false
         type: string
         default: ubuntu-slim
+      eval-runner:
+        description: >-
+          Runner used by the eval harness. Browser evals need a
+          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
+          ship a browser.
+        required: false
+        type: string
+        default: ubuntu-latest
       codeql-runner:
         description: Runner used by CodeQL jobs.
         required: false
@@ -106,28 +114,47 @@ jobs:
       runner: ${{ inputs.test-runner }}
       unit-runner: ${{ inputs.unit-runner }}
       performance-runner: ${{ inputs.performance-runner }}
-      unit-only: ${{ inputs.mode == 'fast' }}
+      # Release pull requests change version metadata only; their content was
+      # audited on the pull requests that merged into main. Integration, E2E,
+      # smoke, and performance lanes stay in the audit and scheduled lanes.
+      unit-only: ${{ inputs.mode == 'fast' || inputs.mode == 'release' }}
       artifact-prefix: ${{ inputs.artifact-prefix }}
     secrets:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
 
   security:
     name: Security
-    if: vars.CI_BILLING_PAUSED != 'true' && (inputs.mode == 'audit' || inputs.mode == 'release')
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
     uses: ./.github/workflows/security.yml
     with:
       runtime-repository: ${{ inputs.runtime-repository }}
       runtime-ref: ${{ inputs.runtime-ref }}
       runner: ${{ inputs.security-runner }}
 
+  eval:
+    name: Eval
+    # Behavior evals cover content changes; the lean release lane skips them
+    # because its diff is version metadata only. The reusable workflow skips
+    # cleanly when the repository has no eval harness.
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
+    uses: ./.github/workflows/eval.yml
+    with:
+      runtime-repository: ${{ inputs.runtime-repository }}
+      runtime-ref: ${{ inputs.runtime-ref }}
+      runner: ${{ inputs.eval-runner }}
+      artifact-prefix: ${{ inputs.artifact-prefix }}
+
   # Aggregate check: "Validation / Gate" (caller job name + this job name).
   # Always evaluates; only jobs required for the mode must have succeeded.
-  # The release tier runs the available CI/test/security suite. Its generated
-  # release diff policy executes as a conditional step below, so no separate
-  # release-policy job renders skipped checks on ordinary pull requests.
+  # The release tier requires the available fast suite; its diff is version
+  # metadata (validated against the release policy below), its content was
+  # audited on the pull requests that merged into main, and the scheduled
+  # audit lane re-covers drift. Its generated release diff policy executes
+  # as a conditional step below, so no separate release-policy job renders
+  # skipped checks on ordinary pull requests.
   gate:
     name: Gate
-    needs: [ci, test, security]
+    needs: [ci, test, security, eval]
     if: vars.CI_BILLING_PAUSED != 'true' && always()
     runs-on: ubuntu-slim
     timeout-minutes: 10
@@ -136,6 +163,7 @@ jobs:
       FOUNDRY_CI: ${{ needs.ci.result }}
       FOUNDRY_TEST: ${{ needs.test.result }}
       FOUNDRY_SECURITY: ${{ needs.security.result }}
+      FOUNDRY_EVAL: ${{ needs.eval.result }}
       # This workflow is selected only for repositories whose canonical
       # configuration explicitly disables unavailable CodeQL. Treat that
       # policy decision as satisfied without registering a skipped PR check.

package/.github/workflows/validation.yml

@@ -42,6 +42,14 @@ on:
         required: false
         type: string
         default: ubuntu-slim
+      eval-runner:
+        description: >-
+          Runner used by the eval harness. Browser evals need a
+          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
+          ship a browser.
+        required: false
+        type: string
+        default: ubuntu-latest
       codeql-runner:
         description: Runner used by CodeQL jobs.
         required: false
@@ -106,14 +114,17 @@ jobs:
       runner: ${{ inputs.test-runner }}
       unit-runner: ${{ inputs.unit-runner }}
       performance-runner: ${{ inputs.performance-runner }}
-      unit-only: ${{ inputs.mode == 'fast' }}
+      # Release pull requests change version metadata only; their content was
+      # audited on the pull requests that merged into main. Integration, E2E,
+      # smoke, and performance lanes stay in the audit and scheduled lanes.
+      unit-only: ${{ inputs.mode == 'fast' || inputs.mode == 'release' }}
       artifact-prefix: ${{ inputs.artifact-prefix }}
     secrets:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
 
   security:
     name: Security
-    if: vars.CI_BILLING_PAUSED != 'true' && (inputs.mode == 'audit' || inputs.mode == 'release')
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
     uses: ./.github/workflows/security.yml
     with:
       runtime-repository: ${{ inputs.runtime-repository }}
@@ -134,15 +145,30 @@ jobs:
       rust-threads: ${{ inputs.rust-threads }}
       rust-max-parallel: ${{ inputs.rust-max-parallel }}
 
+  eval:
+    name: Eval
+    # Behavior evals cover content changes; the lean release lane skips them
+    # because its diff is version metadata only. The reusable workflow skips
+    # cleanly when the repository has no eval harness.
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
+    uses: ./.github/workflows/eval.yml
+    with:
+      runtime-repository: ${{ inputs.runtime-repository }}
+      runtime-ref: ${{ inputs.runtime-ref }}
+      runner: ${{ inputs.eval-runner }}
+      artifact-prefix: ${{ inputs.artifact-prefix }}
+
   # Aggregate check: "Validation / Gate" (caller job name + this job name).
   # Always evaluates; only jobs required for the mode must have succeeded.
-  # The release tier runs the full audit suite so Release Please pull requests
-  # expose no neutral suite checks. Its generated release diff policy executes
+  # The release tier requires the fast suite plus CodeQL: its diff is version
+  # metadata (validated against the release policy below), its content was
+  # audited on the pull requests that merged into main, and the scheduled
+  # audit lane re-covers drift. Its generated release diff policy executes
   # as a conditional step below, so ordinary pull requests still render no
   # skipped release-policy row.
   gate:
     name: Gate
-    needs: [ci, test, security, codeql]
+    needs: [ci, test, security, codeql, eval]
     if: vars.CI_BILLING_PAUSED != 'true' && always()
     runs-on: ubuntu-slim
     timeout-minutes: 10
@@ -152,6 +178,7 @@ jobs:
       FOUNDRY_TEST: ${{ needs.test.result }}
       FOUNDRY_SECURITY: ${{ needs.security.result }}
       FOUNDRY_CODEQL: ${{ needs.codeql.result }}
+      FOUNDRY_EVAL: ${{ needs.eval.result }}
     steps:
       - name: Checkout runtime
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

package/.github/workflows/validation_audit_self-ci.yml

@@ -34,6 +34,7 @@ jobs:
       unit-runner: ubuntu-slim
       performance-runner: ubuntu-latest
       security-runner: ubuntu-slim
+      eval-runner: ubuntu-latest
       codeql-runner: ubuntu-latest
       rust-shards: '["all"]'
       rust-threads: '1'

package/.github/workflows/validation_self-ci.yml

@@ -64,7 +64,12 @@ jobs:
   validation:
     name: Validation
     needs: mode
-    if: vars.CI_BILLING_PAUSED != 'true' && github.event_name == 'pull_request' && github.event.pull_request.draft == false
+    # Bot-authored pull requests (Dependabot and friends) validate only on an
+    # explicit ready transition: dependency pushes never allocate validation
+    # runners by themselves. A human pushing to a bot branch also runs, since
+    # the sender is no longer a bot. Release Please heads are managed by the
+    # release workflow and always validate.
+    if: vars.CI_BILLING_PAUSED != 'true' && github.event_name == 'pull_request' && github.event.pull_request.draft == false && (github.event.pull_request.user.type != 'Bot' || github.event.sender.type != 'Bot' || startsWith(github.event.pull_request.head.ref, 'release-please--branches--main'))
     # Reusable workflows can only maintain or reduce the caller job's scopes.
     # The audit tier needs security-events: write for CodeQL uploads.
     permissions:
@@ -85,6 +90,7 @@ jobs:
       unit-runner: ubuntu-slim
       performance-runner: ubuntu-latest
       security-runner: ubuntu-slim
+      eval-runner: ubuntu-latest
       codeql-runner: ubuntu-latest
       rust-shards: '["all"]'
       rust-threads: '1'

package/AGENTS.md

@@ -17,6 +17,15 @@ They complement `CONTRIBUTING.md`. More specific instructions in nested `AGENTS.
 
 This repository may contain TypeScript, Rust, Python, or any combination of them. Detect the active stack from the files present; do not assume every check applies.
 
+## Contributing back
+
+Consumers are encouraged to help improve this open-source project. Open a small,
+focused pull request for bug fixes, performance improvements, documentation, or
+other narrowly scoped changes. For larger feature requests or architectural
+changes, create an issue first so the proposal can be discussed and scoped.
+Contributions should help make the tool as performant, reliable, and helpful as
+possible for everyone.
+
 ## Read before acting
 
 Before editing:
@@ -149,6 +158,7 @@ node src/runtime.mjs ci unit
 node src/runtime.mjs ci integration
 node src/runtime.mjs ci e2e
 node src/runtime.mjs ci smoke
+node src/runtime.mjs ci eval
 node src/runtime.mjs ci performance
 Security and dependency audits run through the GitHub Security workflow.
 ```

package/CHANGELOG.md

@@ -1,5 +1,82 @@
 # Changelog
 
+## [1.28.2](https://github.com/0xPlayerOne/code-foundry/compare/v1.28.1...v1.28.2) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** pass one qualification report per required node ([#597](https://github.com/0xPlayerOne/code-foundry/issues/597)) ([5ceb92c](https://github.com/0xPlayerOne/code-foundry/commit/5ceb92cab3afc97145ef068e6b5e33399d99b9e5))
+
+## [1.28.1](https://github.com/0xPlayerOne/code-foundry/compare/v1.28.0...v1.28.1) (2026-09-10)
+
+
+### Bug Fixes
+
+* **cloudflare:** remove duplicate deployment records ([#595](https://github.com/0xPlayerOne/code-foundry/issues/595)) ([d9a2a4b](https://github.com/0xPlayerOne/code-foundry/commit/d9a2a4b0363bb9a9ded91d3368bdb7b9b580330f))
+
+## [1.28.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.27.0...v1.28.0) (2026-09-10)
+
+
+### Features
+
+* **release:** qualify only when a release or a stuck draft needs it ([#591](https://github.com/0xPlayerOne/code-foundry/issues/591)) ([4cce045](https://github.com/0xPlayerOne/code-foundry/commit/4cce045228648e6e705ad1e89c59eaf20a72a538))
+
+## [1.27.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.26.0...v1.27.0) (2026-09-10)
+
+
+### Features
+
+* **validation:** downgrade docs-only PRs to fast and gate bot pushes ([#590](https://github.com/0xPlayerOne/code-foundry/issues/590)) ([df774f5](https://github.com/0xPlayerOne/code-foundry/commit/df774f5d1e4a8a6c5f174219eb2b24ef012f6fd3))
+
+## [1.26.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.3...v1.26.0) (2026-09-10)
+
+
+### Features
+
+* **node:** support Node 24 and 26, drop 20 and 22 ([#589](https://github.com/0xPlayerOne/code-foundry/issues/589)) ([ee3e294](https://github.com/0xPlayerOne/code-foundry/commit/ee3e2943889ffd06a8d4cd4ccb75a3f36c46c0fd))
+
+## [1.25.3](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.2...v1.25.3) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** cover the attestation lag in publication retries ([#587](https://github.com/0xPlayerOne/code-foundry/issues/587)) ([7ae5e4b](https://github.com/0xPlayerOne/code-foundry/commit/7ae5e4bb95f68cbf4e2b48778776a0396c877271))
+
+## [1.25.2](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.1...v1.25.2) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** retry integrity verification through the publication consistency window ([#585](https://github.com/0xPlayerOne/code-foundry/issues/585)) ([5b997dd](https://github.com/0xPlayerOne/code-foundry/commit/5b997dda7e272b59a386eeeda115e60ad284f6ce))
+
+## [1.25.1](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.0...v1.25.1) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** wait for the release index before verifying a published release ([#583](https://github.com/0xPlayerOne/code-foundry/issues/583)) ([7823c14](https://github.com/0xPlayerOne/code-foundry/commit/7823c1473d8c5e7c17b039dc755ff5c8d1dba39c))
+
+## [1.25.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.24.0...v1.25.0) (2026-09-10)
+
+
+### Features
+
+* **eval:** run the eval tier as a managed Validation / Eval lane ([#580](https://github.com/0xPlayerOne/code-foundry/issues/580)) ([9414889](https://github.com/0xPlayerOne/code-foundry/commit/94148894da9a627a6bd7c71c846848c198c1dfec))
+
+## [1.24.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.23.0...v1.24.0) (2026-09-10)
+
+
+### Features
+
+* **validation:** run a lean release lane for Release Please pull requests ([#579](https://github.com/0xPlayerOne/code-foundry/issues/579)) ([a38ca75](https://github.com/0xPlayerOne/code-foundry/commit/a38ca75c7da577af8368467b8b026d0ec563fc41))
+
+## [1.23.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.22.1...v1.23.0) (2026-09-09)
+
+
+### Features
+
+* add ci eval tier with shared report contract and budget gate ([#576](https://github.com/0xPlayerOne/code-foundry/issues/576)) ([7d881dd](https://github.com/0xPlayerOne/code-foundry/commit/7d881dd1bc6ace8e48dffff45b1a22a12bdae1c9))
+
 ## [1.22.1](https://github.com/0xPlayerOne/code-foundry/compare/v1.22.0...v1.22.1) (2026-09-09)
 
 

package/README.md

@@ -39,6 +39,9 @@ contract. For normal updates, edit that file and run `npx code-foundry sync`.
   audits, Draft Guard, Draft PR, Release PR, and Release.
 - A deterministic performance lane with ordered command support, stable result
   artifacts, and an optional shared Node package budget profile.
+- A deterministic eval tier (`ci eval`) that runs a repository's behavior-eval
+  harness against a shared report contract with optional budgets, keeping task
+  outcomes comparable across revisions and executors.
 - An opt-in product-quality runner for static sites, web apps, Workers, and
   published packages.
 - A small `.githooks/pre-commit` launcher with language-aware formatting and
@@ -132,7 +135,7 @@ is merged; npm publication is opt-in through `npm_publish: true` and supports
 npm trusted publishing or an `NPM_TOKEN` fallback.
 
 Code Foundry's own release caller is stricter: it qualifies the package across
-Node 20, 22, and 24, stages the exact qualified archive, publishes the
+Node 24 and 26, stages the exact qualified archive, publishes the
 immutable GitHub Release, and publishes that archive through the verified
 publisher. See
 [Consumer qualification](docs/consumer-qualification.md) and [Qualified

package/docs/CONFIGURATION.md

@@ -89,22 +89,29 @@ See [Merge queue validation](merge-queues.md) before enabling it.
 
 ## Validation and quality
 
-| Key                       | Values                                         | Purpose                                                                         |
-| ------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------- |
-| `performance`             | `auto`, `true`, `false`                        | Discover, require, or disable performance checks.                               |
-| `performance_command`     | JSON argv array or array of argv arrays        | Ordered commands for a non-package performance harness.                         |
-| `performance_profile`     | empty or `node-package`                        | Shared package import, memory, archive, and dependency audit.                   |
-| `performance_budget_file` | repository-relative path                       | Budget file for `node-package`; defaults to `performance-package-budgets.json`. |
-| `required_capabilities`   | comma-separated task names                     | Fail closed when a required task or coverage evidence is unavailable.           |
-| `coverage_enforcement`    | `auto`, `required`, `off`                      | Shared coverage-report policy.                                                  |
-| `coverage_minimum`        | `0`–`100`                                      | Minimum percentage; defaults to `80`.                                           |
-| `coverage_metrics`        | `lines`, `functions`, `branches`, `statements` | Metrics checked by the coverage gate.                                           |
-| `coverage_report`         | comma-separated repository paths               | Istanbul JSON summary or LCOV evidence files.                                   |
+| Key                       | Values                                         | Purpose                                                                                                                                         |
+| ------------------------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| `performance`             | `auto`, `true`, `false`                        | Discover, require, or disable performance checks.                                                                                               |
+| `performance_command`     | JSON argv array or array of argv arrays        | Ordered commands for a non-package performance harness.                                                                                         |
+| `performance_profile`     | empty or `node-package`                        | Shared package import, memory, archive, and dependency audit.                                                                                   |
+| `performance_budget_file` | repository-relative path                       | Budget file for `node-package`; defaults to `performance-package-budgets.json`.                                                                 |
+| `eval`                    | `auto`, `true`, `false`                        | Discover, require, or disable the deterministic eval tier.                                                                                      |
+| `eval_command`            | JSON argv array                                | Explicit harness command when there is no `eval` package script.                                                                                |
+| `eval_report_file`        | repository-relative path                       | Report validated against the eval contract; defaults to `eval-results/result.json`.                                                             |
+| `eval_budget_file`        | repository-relative path                       | Optional budget file; defaults to `eval-budgets.json`.                                                                                          |
+| `eval_runner`             | runner label                                   | Runner for the `Validation / Eval` lane; defaults to the repository runner. Browser evals need a Chrome-capable runner such as `ubuntu-latest`. |
+| `required_capabilities`   | comma-separated task names                     | Fail closed when a required task or coverage evidence is unavailable.                                                                           |
+| `coverage_enforcement`    | `auto`, `required`, `off`                      | Shared coverage-report policy.                                                                                                                  |
+| `coverage_minimum`        | `0`–`100`                                      | Minimum percentage; defaults to `80`.                                                                                                           |
+| `coverage_metrics`        | `lines`, `functions`, `branches`, `statements` | Metrics checked by the coverage gate.                                                                                                           |
+| `coverage_report`         | comma-separated repository paths               | Istanbul JSON summary or LCOV evidence files.                                                                                                   |
 
 Supported task capabilities are `format`, `lint`, `type_check`, `build`, `unit`,
-`integration`, `e2e`, `smoke`, and `performance`. `coverage` is a policy
+`integration`, `e2e`, `smoke`, `eval`, and `performance`. `coverage` is a policy
 capability that also requires unit tests. See [Required capabilities and task
-evidence](required-capabilities.md).
+evidence](required-capabilities.md). The eval tier runs the repository's own
+harness against the shared report contract and optional budgets; see
+[Evals](EVALS.md).
 
 The shared performance job discovers `performance:check`, then `perf:check`,
 in JavaScript repositories. Other repositories can provide one command or an

package/docs/EVALS.md

@@ -0,0 +1,139 @@
+# Evals
+
+`ci eval` is the repository's deterministic behavior-evaluation tier. It runs the
+repository's own eval harness and consumes its result against a shared contract,
+so task outcomes stay comparable across revisions and — later — across
+executors (deterministic today, model-agent when a repository adopts one).
+
+Evals are tests' measurement-oriented sibling: tests verify a specified
+behavior (pass/fail); evals record how well the system does at repeated probes
+(success rates, timing percentiles, bounded evidence) and compare those numbers
+against pinned baselines. The `performance` tier is the resource-metric
+special case of the same pattern.
+
+## How the runtime finds your harness
+
+`ci eval` discovers its subject in this order and skips when neither exists:
+
+1. A package script named `eval`.
+2. An explicit `eval_command` (JSON argv array) in `.github/code-foundry.yml`.
+
+Both mechanisms use `eval: auto` by default: the tier runs when a subject is
+present and skips cleanly otherwise. Set `eval: true` to require it (the
+validation policy then treats a missing subject as an error), or `eval: false`
+to disable discovery.
+
+In managed validation the eval tier runs as its own `Validation / Eval` lane
+during audit-mode runs, with the receipt retained as a task artifact. Browser
+eval harnesses need a Chrome-capable runner: set `eval_runner: ubuntu-latest`
+in `.github/code-foundry.yml` (the default when unset is the repository's
+default runner, which may not ship a browser). The lane uploads the eval
+report alongside the task receipt whenever a run fails, so reviewers get the
+measured numbers with the red check. Performance's task profile and gating
+rules apply unchanged; evals are an optional, non-gating tier like
+performance.
+
+| Config key         | Values                            | Meaning                                                          |
+| ------------------ | --------------------------------- | ---------------------------------------------------------------- |
+| `eval`             | `auto` (default), `true`, `false` | Enable, require, or disable the eval tier.                       |
+| `eval_command`     | JSON argv array                   | Explicit harness command when there is no `eval` package script. |
+| `eval_report_file` | repository-relative path          | The report to validate; defaults to `eval-results/result.json`.  |
+| `eval_budget_file` | repository-relative path          | Optional budget file; defaults to `eval-budgets.json`.           |
+
+## The report contract
+
+Your harness writes `eval-results/result.json`. The runtime validates the
+envelope before budgets; a report that violates the contract fails the tier
+with every violation listed (fail closed, typos can never pass silently):
+
+```jsonc
+{
+  "schemaVersion": 1, // contract version, currently 1
+  "revision": "<git sha>", // optional but strongly recommended
+  "dependencyHash": "<sha256>", // optional; pins the dependency set
+  "summary": {
+    "taskCount": 7,
+    "attempts": 7,
+    "passed": 7,
+    "failed": 0,
+    "harnessFailures": 0, // environment broke; not a task regression
+    "successRate": 1.0,
+    "toolCalls": 31,
+    "evidenceErrors": 0,
+    "taskDurationMs": { "count": 7, "mean": 2.1, "p50": 1.8, "p95": 3.0, "max": 3.4 },
+    "startupMs": { "count": 7, "mean": 0.3, "p50": 0.3, "p95": 0.4, "max": 0.4 },
+    "stepDurationMs": { "count": 31, "mean": 0.2, "p50": 0.1, "p95": 0.6, "max": 0.6 },
+  },
+  "tasks": [
+    {
+      "id": "form-submit",
+      "attempts": [
+        {
+          "iteration": 1,
+          "status": "passed",
+          "durationMs": 2.1,
+          "startupMs": 0.3,
+          "steps": [{ "tool": "fill_form", "status": "passed", "durationMs": 0.4 }],
+          "checks": ["fill_form completed"],
+          "metrics": { "formFields": 2 },
+        },
+      ],
+    },
+  ],
+}
+```
+
+Rules the validator enforces:
+
+- `schemaVersion` must match the contract version; `revision`/`dependencyHash`
+  must be strings when present.
+- Summary counts are non-negative integers; `passed + failed` cannot exceed
+  `attempts`; `successRate` is between 0 and 1.
+- `harnessFailures` counts attempts where the environment itself broke (browser
+  never started, zero steps succeeded, cleanup failed). It must not exceed
+  `attempts`. Fix the environment; do not treat these as task regressions.
+- Failed attempts carry a bounded `failure` (non-empty `message`) plus a
+  `failureClass` of `harness` or `task`.
+- `taskDurationMs`, `startupMs`, and `stepDurationMs` are stats objects with
+  `count`, `mean`, `p50`, `p95`, `max` (or bare `{ "count": 0 }`).
+
+A reference implementation ships in
+[`pi-browser-use`](https://github.com/0xPlayerOne/pi-browser-use)
+(`scripts/eval.mjs` + `docs/eval-results.md`): deterministic browser-tool tasks
+that emit this exact envelope.
+
+## Budgets
+
+Create `eval-budgets.json` (or point `eval_budget_file` at another file) to
+gate on the measured numbers:
+
+```json
+{
+  "successRate": 1.0,
+  "stepP95Ms": 1000,
+  "taskP95Ms": 45000,
+  "maxHarnessFailures": 0
+}
+```
+
+Supported budgets: `successRate`, `taskP95Ms`, `startupP95Ms`, `stepP95Ms`,
+`maxHarnessFailures`, `maxEvidenceErrors`, `maxToolCalls`. Unknown keys fail
+closed. When the budget file is absent the tier validates the contract only.
+Budget files are committed configuration; only `eval-results/` is a local
+artifact.
+
+The runtime writes `eval-results/summary.json` with the executed commands, the
+budget outcome, and the artifact list, mirroring the performance summary.
+
+## Determinism rules for eval tasks
+
+- Fixed fixtures and no network. A task's outcome must depend only on the code
+  under test.
+- Bounded evidence: failure text and screenshots are capped; a failing task
+  never produces unbounded output.
+- Cleanup failures are surfaced, never swallowed silently.
+- Every attempt records the revision and dependency hash it ran against.
+- Keep model-in-the-loop runs out of this tier. Reuse the same task IDs in a
+  separate scheduled harness with frozen model/reasoning/judge baselines when
+  you need agent-capability measurement; its variance is why it must never
+  gate pull requests.

package/docs/PUBLISHING.md

@@ -37,7 +37,7 @@ registry version and its provenance link before treating the setup as complete.
 The self-hosted package follows a stricter contract than generated consumer
 callers:
 
-1. Pack the candidate once and qualify that archive across Node 20, 22, and 24.
+1. Pack the candidate once and qualify that archive across Node 24 and 26.
 2. Create a draft GitHub Release and attach the exact qualified archive plus its
    qualification receipt.
 3. Publish the immutable GitHub Release after verifying the tag, source commit,
@@ -50,6 +50,11 @@ the bytes selected from the same workflow run and attempt. See [Consumer
 qualification](consumer-qualification.md) and [Qualified publication](qualified-publication.md)
 for the complete contract, retries, and recovery rules.
 
+The main-push pipeline runs Release Please first, then qualifies only when the
+push created a release or recovery found a stuck draft. Feature merges pay for
+the cheap release-please and recovery probes; the runner-heavy matrix runs on
+release merges, which re-qualify the exact tree they publish.
+
 ## GitHub Releases and GitHub Packages
 
 A GitHub Release is metadata attached to a Git tag. It is independent of npm and

package/docs/WORKFLOWS.md

@@ -27,7 +27,24 @@ ready transition. Release Please version heads are excluded because the
 release workflow owns their state. A separate draft-control caller listens for
 `converted_to_draft` and cancels queued or running pull-request workflows.
 Marking a pull request ready starts validation, and each new commit on a ready
-pull request starts it again for the current head.
+pull request starts it again for the current head. Audit-mode runs additionally
+execute the eval lane (`Validation / Eval`) for repositories that ship an eval
+harness; see [Evals](EVALS.md).
+
+Two waste-avoidance rules keep validation minutes honest without weakening
+confidence:
+
+- **Docs-only pull requests run the fast tier.** When an audit-classified pull
+  request changes only markdown, `docs/`, and license roots, the mode
+  classifier downgrades it to fast. Any code, lockfile, workflow,
+  configuration, or packaging change — or any diff that cannot be computed
+  deterministically — keeps the audit tier. Scheduled and manual audits always
+  run the full tier.
+- **Bot-authored pull requests validate only on ready transitions.**
+  Dependabot and other bot pushes never allocate validation runners; a
+  maintainer marks the (Guard-drafted) pull request ready — or pushes to its
+  branch, which changes the sender — to run validation. Release Please heads
+  are exempt because the release workflow merges them through its own lane.
 
 The separate `validation-audit.yml` caller is pinned to the configured released
 runtime and handles scheduled and manual audits:
@@ -40,9 +57,15 @@ workflow_dispatch:
 
 In the `staging-release` topology, pull requests into `staging` run the fast
 tier, ordinary pull requests into `main` run the full audit tier, and exact
-Release Please pull requests into `main` run the full audit tier plus the
-release-diff policy. This keeps repository rulesets satisfiable for the release
-commit without exposing neutral or skipped suite checks. In the
+Release Please pull requests into `main` run a lean release lane — CI, unit
+tests, and CodeQL plus the release-diff policy. Their diff is version
+metadata only: the content was fully audited on the pull requests that
+merged into `main`, and the scheduled audit lane re-covers drift, so the
+release lane keeps repository rulesets satisfiable without re-running the
+runner-heavy suites. The managed branch rulesets require only the aggregate
+`Validation / Gate` check, so skipped non-required jobs never deadlock the
+release; do not hand-require individual job contexts on release branches.
+In the
 `direct` topology (the default) every pull request targets `main` and runs the
 full audit tier, because there is no integration branch for a fast pass.
 Scheduled and manual runs select the audit tier in both topologies. Draft PR