code-foundry 1.22.1 → 1.25.3

package/.github/workflows/eval.yml

@@ -0,0 +1,116 @@
+name: Code Foundry Eval
+
+on:
+  workflow_call:
+    inputs:
+      runtime-repository:
+        description: Repository containing the Code Foundry runtime.
+        required: false
+        type: string
+        default: 0xPlayerOne/code-foundry
+      runtime-ref:
+        description: Code Foundry runtime tag or ref.
+        required: false
+        type: string
+        default: v1.0.5
+      runner:
+        description: >-
+          Runner used by the eval harness. Browser evals need a
+          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
+          ship a browser.
+        required: false
+        type: string
+        default: ubuntu-latest
+      artifact-prefix:
+        description: Prefix for task receipts; use a unique value when calling this workflow more than once.
+        required: false
+        type: string
+        default: task-result
+
+permissions:
+  contents: read
+
+env:
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+  REPO_FOUNDRY_PROFILE: ${{ vars.REPO_FOUNDRY_PROFILE }}
+  REPO_FOUNDRY_LANGUAGES: ${{ vars.REPO_FOUNDRY_LANGUAGES }}
+  REPO_FOUNDRY_FEATURES: ${{ vars.REPO_FOUNDRY_FEATURES }}
+  REPO_FOUNDRY_PACKAGE_MANAGER: ${{ vars.REPO_FOUNDRY_PACKAGE_MANAGER }}
+  REPO_FOUNDRY_RUNNER: ${{ vars.REPO_FOUNDRY_RUNNER }}
+  REPO_FOUNDRY_CACHE_PACKAGES: ${{ vars.REPO_FOUNDRY_CACHE_PACKAGES || 'auto' }}
+  REPO_FOUNDRY_CACHE_BUILD: ${{ vars.REPO_FOUNDRY_CACHE_BUILD || 'auto' }}
+
+concurrency:
+  group: code-foundry-eval-${{ github.event_name }}-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.event.pull_request.head.ref || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  eval:
+    name: Eval
+    if: vars.CI_BILLING_PAUSED != 'true'
+    runs-on: ${{ inputs.runner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Runtime
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+          repository: ${{ inputs.runtime-repository }}
+          ref: ${{ inputs.runtime-ref }}
+          path: .github/.code-foundry
+          sparse-checkout: |
+            .github/actions
+            src/lib
+            src/runtime-core.mjs
+            src/runtime.mjs
+      - name: Install runtime
+        run: |
+          mkdir -p .github/actions
+          mv .github/.code-foundry "$RUNNER_TEMP/code-foundry"
+          cp -R "$RUNNER_TEMP/code-foundry/.github/actions/." .github/actions/
+      - name: Detect
+        id: applicability
+        run: node "$RUNNER_TEMP/code-foundry/src/runtime.mjs" ci should_run eval >> "$GITHUB_OUTPUT"
+      - name: Setup
+        if: steps.applicability.outputs.applicable == 'true'
+        uses: ./.github/actions/setup
+        with:
+          # Eval harnesses run through the repository's own package scripts and
+          # need their dependencies installed before the task executes.
+          install: 'true'
+          cache-build: 'true'
+          task: eval
+      - name: Eval
+        id: execute
+        if: steps.applicability.outputs.applicable == 'true'
+        run: node "$RUNNER_TEMP/code-foundry/src/runtime.mjs" ci eval
+      - name: Retain task receipt
+        if: >-
+          always() &&
+          (steps.execute.outcome == 'success' || steps.execute.outcome == 'failure' ||
+          steps.applicability.outputs.applicable == 'false') &&
+          hashFiles('.code-foundry/results/eval.json') != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ${{ inputs.artifact-prefix }}-${{ github.run_id }}-${{ github.run_attempt }}-eval
+          path: .code-foundry/results/eval.json
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 14
+      - name: Retain eval report
+        if: >-
+          always() &&
+          steps.execute.outcome == 'failure' &&
+          hashFiles('eval-results/result.json') != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ${{ inputs.artifact-prefix }}-${{ github.run_id }}-${{ github.run_attempt }}-eval-report
+          path: |
+            eval-results/result.json
+            eval-results/summary.json
+          if-no-files-found: error
+          retention-days: 14

package/.github/workflows/validation-no-codeql.yml

@@ -42,6 +42,14 @@ on:
         required: false
         type: string
         default: ubuntu-slim
+      eval-runner:
+        description: >-
+          Runner used by the eval harness. Browser evals need a
+          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
+          ship a browser.
+        required: false
+        type: string
+        default: ubuntu-latest
       codeql-runner:
         description: Runner used by CodeQL jobs.
         required: false
@@ -106,28 +114,47 @@ jobs:
       runner: ${{ inputs.test-runner }}
       unit-runner: ${{ inputs.unit-runner }}
       performance-runner: ${{ inputs.performance-runner }}
-      unit-only: ${{ inputs.mode == 'fast' }}
+      # Release pull requests change version metadata only; their content was
+      # audited on the pull requests that merged into main. Integration, E2E,
+      # smoke, and performance lanes stay in the audit and scheduled lanes.
+      unit-only: ${{ inputs.mode == 'fast' || inputs.mode == 'release' }}
       artifact-prefix: ${{ inputs.artifact-prefix }}
     secrets:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
 
   security:
     name: Security
-    if: vars.CI_BILLING_PAUSED != 'true' && (inputs.mode == 'audit' || inputs.mode == 'release')
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
     uses: ./.github/workflows/security.yml
     with:
       runtime-repository: ${{ inputs.runtime-repository }}
       runtime-ref: ${{ inputs.runtime-ref }}
       runner: ${{ inputs.security-runner }}
 
+  eval:
+    name: Eval
+    # Behavior evals cover content changes; the lean release lane skips them
+    # because its diff is version metadata only. The reusable workflow skips
+    # cleanly when the repository has no eval harness.
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
+    uses: ./.github/workflows/eval.yml
+    with:
+      runtime-repository: ${{ inputs.runtime-repository }}
+      runtime-ref: ${{ inputs.runtime-ref }}
+      runner: ${{ inputs.eval-runner }}
+      artifact-prefix: ${{ inputs.artifact-prefix }}
+
   # Aggregate check: "Validation / Gate" (caller job name + this job name).
   # Always evaluates; only jobs required for the mode must have succeeded.
-  # The release tier runs the available CI/test/security suite. Its generated
-  # release diff policy executes as a conditional step below, so no separate
-  # release-policy job renders skipped checks on ordinary pull requests.
+  # The release tier requires the available fast suite; its diff is version
+  # metadata (validated against the release policy below), its content was
+  # audited on the pull requests that merged into main, and the scheduled
+  # audit lane re-covers drift. Its generated release diff policy executes
+  # as a conditional step below, so no separate release-policy job renders
+  # skipped checks on ordinary pull requests.
   gate:
     name: Gate
-    needs: [ci, test, security]
+    needs: [ci, test, security, eval]
     if: vars.CI_BILLING_PAUSED != 'true' && always()
     runs-on: ubuntu-slim
     timeout-minutes: 10
@@ -136,6 +163,7 @@ jobs:
       FOUNDRY_CI: ${{ needs.ci.result }}
       FOUNDRY_TEST: ${{ needs.test.result }}
       FOUNDRY_SECURITY: ${{ needs.security.result }}
+      FOUNDRY_EVAL: ${{ needs.eval.result }}
       # This workflow is selected only for repositories whose canonical
       # configuration explicitly disables unavailable CodeQL. Treat that
       # policy decision as satisfied without registering a skipped PR check.

package/.github/workflows/validation.yml

@@ -42,6 +42,14 @@ on:
         required: false
         type: string
         default: ubuntu-slim
+      eval-runner:
+        description: >-
+          Runner used by the eval harness. Browser evals need a
+          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
+          ship a browser.
+        required: false
+        type: string
+        default: ubuntu-latest
       codeql-runner:
         description: Runner used by CodeQL jobs.
         required: false
@@ -106,14 +114,17 @@ jobs:
       runner: ${{ inputs.test-runner }}
       unit-runner: ${{ inputs.unit-runner }}
       performance-runner: ${{ inputs.performance-runner }}
-      unit-only: ${{ inputs.mode == 'fast' }}
+      # Release pull requests change version metadata only; their content was
+      # audited on the pull requests that merged into main. Integration, E2E,
+      # smoke, and performance lanes stay in the audit and scheduled lanes.
+      unit-only: ${{ inputs.mode == 'fast' || inputs.mode == 'release' }}
       artifact-prefix: ${{ inputs.artifact-prefix }}
     secrets:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
 
   security:
     name: Security
-    if: vars.CI_BILLING_PAUSED != 'true' && (inputs.mode == 'audit' || inputs.mode == 'release')
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
     uses: ./.github/workflows/security.yml
     with:
       runtime-repository: ${{ inputs.runtime-repository }}
@@ -134,15 +145,30 @@ jobs:
       rust-threads: ${{ inputs.rust-threads }}
       rust-max-parallel: ${{ inputs.rust-max-parallel }}
 
+  eval:
+    name: Eval
+    # Behavior evals cover content changes; the lean release lane skips them
+    # because its diff is version metadata only. The reusable workflow skips
+    # cleanly when the repository has no eval harness.
+    if: vars.CI_BILLING_PAUSED != 'true' && inputs.mode == 'audit'
+    uses: ./.github/workflows/eval.yml
+    with:
+      runtime-repository: ${{ inputs.runtime-repository }}
+      runtime-ref: ${{ inputs.runtime-ref }}
+      runner: ${{ inputs.eval-runner }}
+      artifact-prefix: ${{ inputs.artifact-prefix }}
+
   # Aggregate check: "Validation / Gate" (caller job name + this job name).
   # Always evaluates; only jobs required for the mode must have succeeded.
-  # The release tier runs the full audit suite so Release Please pull requests
-  # expose no neutral suite checks. Its generated release diff policy executes
+  # The release tier requires the fast suite plus CodeQL: its diff is version
+  # metadata (validated against the release policy below), its content was
+  # audited on the pull requests that merged into main, and the scheduled
+  # audit lane re-covers drift. Its generated release diff policy executes
   # as a conditional step below, so ordinary pull requests still render no
   # skipped release-policy row.
   gate:
     name: Gate
-    needs: [ci, test, security, codeql]
+    needs: [ci, test, security, codeql, eval]
     if: vars.CI_BILLING_PAUSED != 'true' && always()
     runs-on: ubuntu-slim
     timeout-minutes: 10
@@ -152,6 +178,7 @@ jobs:
       FOUNDRY_TEST: ${{ needs.test.result }}
       FOUNDRY_SECURITY: ${{ needs.security.result }}
       FOUNDRY_CODEQL: ${{ needs.codeql.result }}
+      FOUNDRY_EVAL: ${{ needs.eval.result }}
     steps:
       - name: Checkout runtime
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

package/.github/workflows/validation_audit_self-ci.yml

@@ -34,6 +34,7 @@ jobs:
       unit-runner: ubuntu-slim
       performance-runner: ubuntu-latest
       security-runner: ubuntu-slim
+      eval-runner: ubuntu-latest
       codeql-runner: ubuntu-latest
       rust-shards: '["all"]'
       rust-threads: '1'

package/.github/workflows/validation_self-ci.yml

@@ -85,6 +85,7 @@ jobs:
       unit-runner: ubuntu-slim
       performance-runner: ubuntu-latest
       security-runner: ubuntu-slim
+      eval-runner: ubuntu-latest
       codeql-runner: ubuntu-latest
       rust-shards: '["all"]'
       rust-threads: '1'

package/AGENTS.md

@@ -149,6 +149,7 @@ node src/runtime.mjs ci unit
 node src/runtime.mjs ci integration
 node src/runtime.mjs ci e2e
 node src/runtime.mjs ci smoke
+node src/runtime.mjs ci eval
 node src/runtime.mjs ci performance
 Security and dependency audits run through the GitHub Security workflow.
 ```

package/CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## [1.25.3](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.2...v1.25.3) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** cover the attestation lag in publication retries ([#587](https://github.com/0xPlayerOne/code-foundry/issues/587)) ([7ae5e4b](https://github.com/0xPlayerOne/code-foundry/commit/7ae5e4bb95f68cbf4e2b48778776a0396c877271))
+
+## [1.25.2](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.1...v1.25.2) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** retry integrity verification through the publication consistency window ([#585](https://github.com/0xPlayerOne/code-foundry/issues/585)) ([5b997dd](https://github.com/0xPlayerOne/code-foundry/commit/5b997dda7e272b59a386eeeda115e60ad284f6ce))
+
+## [1.25.1](https://github.com/0xPlayerOne/code-foundry/compare/v1.25.0...v1.25.1) (2026-09-10)
+
+
+### Bug Fixes
+
+* **release:** wait for the release index before verifying a published release ([#583](https://github.com/0xPlayerOne/code-foundry/issues/583)) ([7823c14](https://github.com/0xPlayerOne/code-foundry/commit/7823c1473d8c5e7c17b039dc755ff5c8d1dba39c))
+
+## [1.25.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.24.0...v1.25.0) (2026-09-10)
+
+
+### Features
+
+* **eval:** run the eval tier as a managed Validation / Eval lane ([#580](https://github.com/0xPlayerOne/code-foundry/issues/580)) ([9414889](https://github.com/0xPlayerOne/code-foundry/commit/94148894da9a627a6bd7c71c846848c198c1dfec))
+
+## [1.24.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.23.0...v1.24.0) (2026-09-10)
+
+
+### Features
+
+* **validation:** run a lean release lane for Release Please pull requests ([#579](https://github.com/0xPlayerOne/code-foundry/issues/579)) ([a38ca75](https://github.com/0xPlayerOne/code-foundry/commit/a38ca75c7da577af8368467b8b026d0ec563fc41))
+
+## [1.23.0](https://github.com/0xPlayerOne/code-foundry/compare/v1.22.1...v1.23.0) (2026-09-09)
+
+
+### Features
+
+* add ci eval tier with shared report contract and budget gate ([#576](https://github.com/0xPlayerOne/code-foundry/issues/576)) ([7d881dd](https://github.com/0xPlayerOne/code-foundry/commit/7d881dd1bc6ace8e48dffff45b1a22a12bdae1c9))
+
 ## [1.22.1](https://github.com/0xPlayerOne/code-foundry/compare/v1.22.0...v1.22.1) (2026-09-09)
 
 

package/README.md

@@ -39,6 +39,9 @@ contract. For normal updates, edit that file and run `npx code-foundry sync`.
   audits, Draft Guard, Draft PR, Release PR, and Release.
 - A deterministic performance lane with ordered command support, stable result
   artifacts, and an optional shared Node package budget profile.
+- A deterministic eval tier (`ci eval`) that runs a repository's behavior-eval
+  harness against a shared report contract with optional budgets, keeping task
+  outcomes comparable across revisions and executors.
 - An opt-in product-quality runner for static sites, web apps, Workers, and
   published packages.
 - A small `.githooks/pre-commit` launcher with language-aware formatting and

package/docs/CONFIGURATION.md

@@ -89,22 +89,29 @@ See [Merge queue validation](merge-queues.md) before enabling it.
 
 ## Validation and quality
 
-| Key                       | Values                                         | Purpose                                                                         |
-| ------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------- |
-| `performance`             | `auto`, `true`, `false`                        | Discover, require, or disable performance checks.                               |
-| `performance_command`     | JSON argv array or array of argv arrays        | Ordered commands for a non-package performance harness.                         |
-| `performance_profile`     | empty or `node-package`                        | Shared package import, memory, archive, and dependency audit.                   |
-| `performance_budget_file` | repository-relative path                       | Budget file for `node-package`; defaults to `performance-package-budgets.json`. |
-| `required_capabilities`   | comma-separated task names                     | Fail closed when a required task or coverage evidence is unavailable.           |
-| `coverage_enforcement`    | `auto`, `required`, `off`                      | Shared coverage-report policy.                                                  |
-| `coverage_minimum`        | `0`–`100`                                      | Minimum percentage; defaults to `80`.                                           |
-| `coverage_metrics`        | `lines`, `functions`, `branches`, `statements` | Metrics checked by the coverage gate.                                           |
-| `coverage_report`         | comma-separated repository paths               | Istanbul JSON summary or LCOV evidence files.                                   |
+| Key                       | Values                                         | Purpose                                                                                                                                         |
+| ------------------------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| `performance`             | `auto`, `true`, `false`                        | Discover, require, or disable performance checks.                                                                                               |
+| `performance_command`     | JSON argv array or array of argv arrays        | Ordered commands for a non-package performance harness.                                                                                         |
+| `performance_profile`     | empty or `node-package`                        | Shared package import, memory, archive, and dependency audit.                                                                                   |
+| `performance_budget_file` | repository-relative path                       | Budget file for `node-package`; defaults to `performance-package-budgets.json`.                                                                 |
+| `eval`                    | `auto`, `true`, `false`                        | Discover, require, or disable the deterministic eval tier.                                                                                      |
+| `eval_command`            | JSON argv array                                | Explicit harness command when there is no `eval` package script.                                                                                |
+| `eval_report_file`        | repository-relative path                       | Report validated against the eval contract; defaults to `eval-results/result.json`.                                                             |
+| `eval_budget_file`        | repository-relative path                       | Optional budget file; defaults to `eval-budgets.json`.                                                                                          |
+| `eval_runner`             | runner label                                   | Runner for the `Validation / Eval` lane; defaults to the repository runner. Browser evals need a Chrome-capable runner such as `ubuntu-latest`. |
+| `required_capabilities`   | comma-separated task names                     | Fail closed when a required task or coverage evidence is unavailable.                                                                           |
+| `coverage_enforcement`    | `auto`, `required`, `off`                      | Shared coverage-report policy.                                                                                                                  |
+| `coverage_minimum`        | `0`–`100`                                      | Minimum percentage; defaults to `80`.                                                                                                           |
+| `coverage_metrics`        | `lines`, `functions`, `branches`, `statements` | Metrics checked by the coverage gate.                                                                                                           |
+| `coverage_report`         | comma-separated repository paths               | Istanbul JSON summary or LCOV evidence files.                                                                                                   |
 
 Supported task capabilities are `format`, `lint`, `type_check`, `build`, `unit`,
-`integration`, `e2e`, `smoke`, and `performance`. `coverage` is a policy
+`integration`, `e2e`, `smoke`, `eval`, and `performance`. `coverage` is a policy
 capability that also requires unit tests. See [Required capabilities and task
-evidence](required-capabilities.md).
+evidence](required-capabilities.md). The eval tier runs the repository's own
+harness against the shared report contract and optional budgets; see
+[Evals](EVALS.md).
 
 The shared performance job discovers `performance:check`, then `perf:check`,
 in JavaScript repositories. Other repositories can provide one command or an

package/docs/EVALS.md

@@ -0,0 +1,139 @@
+# Evals
+
+`ci eval` is the repository's deterministic behavior-evaluation tier. It runs the
+repository's own eval harness and consumes its result against a shared contract,
+so task outcomes stay comparable across revisions and — later — across
+executors (deterministic today, model-agent when a repository adopts one).
+
+Evals are tests' measurement-oriented sibling: tests verify a specified
+behavior (pass/fail); evals record how well the system does at repeated probes
+(success rates, timing percentiles, bounded evidence) and compare those numbers
+against pinned baselines. The `performance` tier is the resource-metric
+special case of the same pattern.
+
+## How the runtime finds your harness
+
+`ci eval` discovers its subject in this order and skips when neither exists:
+
+1. A package script named `eval`.
+2. An explicit `eval_command` (JSON argv array) in `.github/code-foundry.yml`.
+
+Both mechanisms use `eval: auto` by default: the tier runs when a subject is
+present and skips cleanly otherwise. Set `eval: true` to require it (the
+validation policy then treats a missing subject as an error), or `eval: false`
+to disable discovery.
+
+In managed validation the eval tier runs as its own `Validation / Eval` lane
+during audit-mode runs, with the receipt retained as a task artifact. Browser
+eval harnesses need a Chrome-capable runner: set `eval_runner: ubuntu-latest`
+in `.github/code-foundry.yml` (the default when unset is the repository's
+default runner, which may not ship a browser). The lane uploads the eval
+report alongside the task receipt whenever a run fails, so reviewers get the
+measured numbers with the red check. Performance's task profile and gating
+rules apply unchanged; evals are an optional, non-gating tier like
+performance.
+
+| Config key         | Values                            | Meaning                                                          |
+| ------------------ | --------------------------------- | ---------------------------------------------------------------- |
+| `eval`             | `auto` (default), `true`, `false` | Enable, require, or disable the eval tier.                       |
+| `eval_command`     | JSON argv array                   | Explicit harness command when there is no `eval` package script. |
+| `eval_report_file` | repository-relative path          | The report to validate; defaults to `eval-results/result.json`.  |
+| `eval_budget_file` | repository-relative path          | Optional budget file; defaults to `eval-budgets.json`.           |
+
+## The report contract
+
+Your harness writes `eval-results/result.json`. The runtime validates the
+envelope before budgets; a report that violates the contract fails the tier
+with every violation listed (fail closed, typos can never pass silently):
+
+```jsonc
+{
+  "schemaVersion": 1, // contract version, currently 1
+  "revision": "<git sha>", // optional but strongly recommended
+  "dependencyHash": "<sha256>", // optional; pins the dependency set
+  "summary": {
+    "taskCount": 7,
+    "attempts": 7,
+    "passed": 7,
+    "failed": 0,
+    "harnessFailures": 0, // environment broke; not a task regression
+    "successRate": 1.0,
+    "toolCalls": 31,
+    "evidenceErrors": 0,
+    "taskDurationMs": { "count": 7, "mean": 2.1, "p50": 1.8, "p95": 3.0, "max": 3.4 },
+    "startupMs": { "count": 7, "mean": 0.3, "p50": 0.3, "p95": 0.4, "max": 0.4 },
+    "stepDurationMs": { "count": 31, "mean": 0.2, "p50": 0.1, "p95": 0.6, "max": 0.6 },
+  },
+  "tasks": [
+    {
+      "id": "form-submit",
+      "attempts": [
+        {
+          "iteration": 1,
+          "status": "passed",
+          "durationMs": 2.1,
+          "startupMs": 0.3,
+          "steps": [{ "tool": "fill_form", "status": "passed", "durationMs": 0.4 }],
+          "checks": ["fill_form completed"],
+          "metrics": { "formFields": 2 },
+        },
+      ],
+    },
+  ],
+}
+```
+
+Rules the validator enforces:
+
+- `schemaVersion` must match the contract version; `revision`/`dependencyHash`
+  must be strings when present.
+- Summary counts are non-negative integers; `passed + failed` cannot exceed
+  `attempts`; `successRate` is between 0 and 1.
+- `harnessFailures` counts attempts where the environment itself broke (browser
+  never started, zero steps succeeded, cleanup failed). It must not exceed
+  `attempts`. Fix the environment; do not treat these as task regressions.
+- Failed attempts carry a bounded `failure` (non-empty `message`) plus a
+  `failureClass` of `harness` or `task`.
+- `taskDurationMs`, `startupMs`, and `stepDurationMs` are stats objects with
+  `count`, `mean`, `p50`, `p95`, `max` (or bare `{ "count": 0 }`).
+
+A reference implementation ships in
+[`pi-browser-use`](https://github.com/0xPlayerOne/pi-browser-use)
+(`scripts/eval.mjs` + `docs/eval-results.md`): deterministic browser-tool tasks
+that emit this exact envelope.
+
+## Budgets
+
+Create `eval-budgets.json` (or point `eval_budget_file` at another file) to
+gate on the measured numbers:
+
+```json
+{
+  "successRate": 1.0,
+  "stepP95Ms": 1000,
+  "taskP95Ms": 45000,
+  "maxHarnessFailures": 0
+}
+```
+
+Supported budgets: `successRate`, `taskP95Ms`, `startupP95Ms`, `stepP95Ms`,
+`maxHarnessFailures`, `maxEvidenceErrors`, `maxToolCalls`. Unknown keys fail
+closed. When the budget file is absent the tier validates the contract only.
+Budget files are committed configuration; only `eval-results/` is a local
+artifact.
+
+The runtime writes `eval-results/summary.json` with the executed commands, the
+budget outcome, and the artifact list, mirroring the performance summary.
+
+## Determinism rules for eval tasks
+
+- Fixed fixtures and no network. A task's outcome must depend only on the code
+  under test.
+- Bounded evidence: failure text and screenshots are capped; a failing task
+  never produces unbounded output.
+- Cleanup failures are surfaced, never swallowed silently.
+- Every attempt records the revision and dependency hash it ran against.
+- Keep model-in-the-loop runs out of this tier. Reuse the same task IDs in a
+  separate scheduled harness with frozen model/reasoning/judge baselines when
+  you need agent-capability measurement; its variance is why it must never
+  gate pull requests.

package/docs/WORKFLOWS.md

@@ -27,7 +27,9 @@ ready transition. Release Please version heads are excluded because the
 release workflow owns their state. A separate draft-control caller listens for
 `converted_to_draft` and cancels queued or running pull-request workflows.
 Marking a pull request ready starts validation, and each new commit on a ready
-pull request starts it again for the current head.
+pull request starts it again for the current head. Audit-mode runs additionally
+execute the eval lane (`Validation / Eval`) for repositories that ship an eval
+harness; see [Evals](EVALS.md).
 
 The separate `validation-audit.yml` caller is pinned to the configured released
 runtime and handles scheduled and manual audits:
@@ -40,9 +42,15 @@ workflow_dispatch:
 
 In the `staging-release` topology, pull requests into `staging` run the fast
 tier, ordinary pull requests into `main` run the full audit tier, and exact
-Release Please pull requests into `main` run the full audit tier plus the
-release-diff policy. This keeps repository rulesets satisfiable for the release
-commit without exposing neutral or skipped suite checks. In the
+Release Please pull requests into `main` run a lean release lane — CI, unit
+tests, and CodeQL plus the release-diff policy. Their diff is version
+metadata only: the content was fully audited on the pull requests that
+merged into `main`, and the scheduled audit lane re-covers drift, so the
+release lane keeps repository rulesets satisfiable without re-running the
+runner-heavy suites. The managed branch rulesets require only the aggregate
+`Validation / Gate` check, so skipped non-required jobs never deadlock the
+release; do not hand-require individual job contexts on release branches.
+In the
 `direct` topology (the default) every pull request targets `main` and runs the
 full audit tier, because there is no integration branch for a fast pass.
 Scheduled and manual runs select the audit tier in both topologies. Draft PR

package/docs/required-capabilities.md

@@ -5,8 +5,9 @@ Optional task discovery remains available. Configure scalar values in
 `.github/code-foundry.yml`:
 
 ```yaml
-required_capabilities: type_check,unit,e2e,performance,coverage
+required_capabilities: type_check,unit,e2e,eval,performance,coverage
 performance: true
+eval: true
 coverage_enforcement: required
 coverage_minimum: 80
 coverage_metrics: lines,branches
@@ -14,10 +15,11 @@ coverage_report: coverage/coverage-summary.json
 ```
 
 Supported task capabilities are `format`, `lint`, `type_check`, `build`, `unit`,
-`integration`, `e2e`, `smoke`, and `performance`. `coverage` additionally requires
+`integration`, `e2e`, `smoke`, `eval`, and `performance`. `coverage` additionally requires
 unit tests. Unknown names, contradictory requirements, and invalid thresholds
-are errors. `performance: true` means required, not merely enabled when a
-script happens to exist. Use `performance: auto` to retain optional discovery.
+are errors. `performance: true` and `eval: true` mean required, not merely
+enabled when a script happens to exist. Use `performance: auto` or
+`eval: auto` to retain optional discovery.
 
 The public `src/runtime.mjs` entrypoint delegates ecosystem execution to the
 private `src/runtime-core.mjs`. Keep both files and `src/lib` when vendoring the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "code-foundry",
-  "version": "1.22.1",
+  "version": "1.25.3",
   "description": "A fast, language-aware repository factory for agent-ready workflows, testing, security, and release automation.",
   "homepage": "https://github.com/0xPlayerOne/code-foundry#readme",
   "bugs": {