code-foundry 1.20.2 → 1.25.3

package/src/commands/qualified-publication.mjs

@@ -46,6 +46,24 @@ export function runCommand(command, args, cwd) {
   )
   return result.stdout
 }
+
+/** Default tolerant runner: GitHub CLI failures surface as retryable misses
+ * instead of aborting the publication flow.
+ * @param {string} command
+ * @param {string[]} args
+ * @param {string} [cwd]
+ * @returns {{status: number, stdout: string}}
+ */
+function defaultRunTolerant(command, args, cwd) {
+  const result = spawnSync(command, args, {
+    cwd,
+    encoding: 'utf8',
+    timeout: 180_000,
+    maxBuffer: 8 * 1024 * 1024,
+    env: { ...process.env, GH_HOST: 'github.com', GH_PROMPT_DISABLED: '1' },
+  })
+  return { status: result.error ? 1 : (result.status ?? 1), stdout: result.stdout ?? '' }
+}
 /** @param {Candidate} candidate */
 export function validateCandidate(candidate) {
   ensure(
@@ -225,10 +243,50 @@ export async function publishQualifiedArchive(candidate, reports, adapters = {})
   return { ...before, status: 'published' }
 }
 
+/** @typedef {(command:string, args:string[], cwd?:string) => {status: number, stdout: string}} RunTolerant */
+
+/** Poll the releases-by-tag endpoint through its eventual-consistency window.
+ * Immediately after `gh release edit --draft=false` the endpoint can still
+ * return 404; a strict verification in that window aborts an otherwise
+ * healthy publication. Bounded retries keep the flow strict: it only proceeds
+ * once the endpoint serves the published release with the exact tag.
+ * @param {Candidate} candidate
+ * @param {{runTolerant?: RunTolerant, delay?: (ms:number)=>Promise<void>, attempts?: number}} [adapters]
+ * @returns {Promise<Record<string, any>>}
+ */
+async function waitForPublishedRelease(candidate, adapters = {}) {
+  const runTolerant = adapters.runTolerant ?? defaultRunTolerant
+  const delay = adapters.delay ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const attempts = adapters.attempts ?? 15
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const result = runTolerant('gh', [
+      'api',
+      '--hostname',
+      'github.com',
+      `repos/${candidate.repository}/releases/tags/${encodeURIComponent(candidate.tag)}`,
+    ])
+    if (result.status === 0) {
+      /** @type {Record<string, any> | null} */
+      let release = null
+      try {
+        release = JSON.parse(result.stdout)
+      } catch {
+        release = null
+      }
+      if (release?.draft === false && release?.tag_name === candidate.tag) return release
+    }
+    if (attempt === attempts) break
+    await delay(2_000)
+  }
+  throw new Error(
+    'The published release was not retrievable via the tag endpoint within the consistency window'
+  )
+}
+
 /** Stage only a previously-created draft: never move tags, overwrite assets, or
  * treat an API/permissions failure as an absent release. Authentication needs
  * immutable-settings read and release write; no setting is mutated.
- * @param {Candidate} candidate @param {string[]} reports @param {{run?:Run, verify?:Verify}} [adapters]
+ * @param {Candidate} candidate @param {string[]} reports @param {{run?:Run, runTolerant?:RunTolerant, delay?:(ms:number)=>Promise<void>, attempts?:number, verify?:Verify}} [adapters]
  */
 export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
   const run = adapters.run ?? runCommand
@@ -252,15 +310,28 @@ export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
     )
   }
   checkTag()
-  const endpoint = `${prefix}/releases/tags/${encodeURIComponent(candidate.tag)}`
-  const release = api(endpoint)
-  ensure(
-    Number.isSafeInteger(release.id) &&
-      release.draft === true &&
-      release.tag_name === candidate.tag &&
-      Array.isArray(release.assets),
-    'An existing draft release with the exact tag is required'
-  )
+  const listEndpoint = `${prefix}/releases?per_page=100`
+  const findDraftRelease = () => {
+    const pages = JSON.parse(
+      run('gh', ['api', '--hostname', 'github.com', '--paginate', '--slurp', listEndpoint])
+    )
+    ensure(
+      Array.isArray(pages) && pages.every((page) => Array.isArray(page)),
+      'GitHub returned an invalid release list'
+    )
+    const release = pages.flat().find((entry) => entry?.tag_name === candidate.tag)
+    ensure(
+      Number.isSafeInteger(release?.id) &&
+        release.draft === true &&
+        release.tag_name === candidate.tag &&
+        Array.isArray(release.assets),
+      'An existing draft release with the exact tag is required'
+    )
+    return release
+  }
+  // GitHub's get-by-tag endpoint does not return draft releases. Enumerate the
+  // authenticated release list instead, including all pages, before any write.
+  const release = findDraftRelease()
   const temporary = mkdtempSync(join(tmpdir(), 'foundry-release-receipt-'))
   try {
     const receipt = join(temporary, 'qualification.json')
@@ -288,7 +359,7 @@ export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
           candidate.repository,
         ])
     }
-    const uploaded = api(endpoint)
+    const uploaded = findDraftRelease()
     ensure(
       uploaded.id === release.id && uploaded.draft === true && uploaded.tag_name === candidate.tag,
       'Draft changed during staging'
@@ -319,7 +390,27 @@ export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
       '--repo',
       candidate.repository,
     ])
-    const identity = await (adapters.verify ?? verifier)(candidate)
+    // GitHub's release index is eventually consistent right after a draft is
+    // published; a strict verification a second later can observe a 404 and
+    // abort the publication even though the release is live. Poll until the
+    // index serves the release, then verify; the verification cycle itself
+    // retries through the same window and still fails closed in the end.
+    await waitForPublishedRelease(candidate, {
+      runTolerant: adapters.runTolerant,
+      delay: adapters.delay,
+      attempts: 10,
+    })
+    /** @type {Record<string, any>} */
+    let identity
+    for (let attempt = 1; ; attempt += 1) {
+      try {
+        identity = await (adapters.verify ?? verifier)(candidate)
+        break
+      } catch (error) {
+        if (attempt >= 10) throw error
+        await (adapters.delay ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(5_000)
+      }
+    }
     ensure(
       identity.status === 'passed' &&
         identity.immutable === true &&

package/src/commands/release-integrity.mjs

@@ -7,7 +7,7 @@ import { basename, isAbsolute, relative, resolve, sep } from 'node:path'
 import { spawnSync } from 'node:child_process'
 import { pathToFileURL } from 'node:url'
 
-/** @typedef {(argv: string[]) => {status: number|null, stdout: string}} Runner */
+/** @typedef {(argv: string[]) => {status: number|null, stdout: string, stderr?: string}} Runner */
 
 /** @type {Runner} */
 export function runGh(argv) {
@@ -18,7 +18,7 @@ export function runGh(argv) {
     env: { ...process.env, GH_PROMPT_DISABLED: '1', GH_HOST: 'github.com' },
   })
   if (result.error) throw new Error(`GitHub CLI unavailable: ${result.error.name}`)
-  return { status: result.status, stdout: result.stdout ?? '' }
+  return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }
 }
 
 /** @param {string} repository */
@@ -56,7 +56,7 @@ function json(run, argv) {
   const result = run(argv)
   if (result.status !== 0)
     throw new Error(
-      `GitHub verification command failed (${result.status ?? 'signal'}); state is unverified`
+      `GitHub verification command failed (${result.status ?? 'signal'}); state is unverified: ${String(result.stderr ?? '').slice(0, 300) || 'no stderr captured'}`
     )
   try {
     return JSON.parse(result.stdout)
@@ -140,6 +140,15 @@ function tagCommit(run, repository, tag) {
   return object.sha
 }
 
+/** Resolve a release tag without requiring the release itself to be published.
+ * @param {string} repository @param {string} tag @param {Runner} [run]
+ */
+export function resolveTagCommit(repository, tag, run = runGh) {
+  validateRepository(repository)
+  validateTag(tag)
+  return tagCommit(run, repository, tag)
+}
+
 /** GitHub CLI performs signature/attestation verification, not merely a metadata check.
  * @param {{repository: string, tag: string, root?: string, assets?: string[], expectedSha?: string}} options
  * @param {Runner} [run]
@@ -188,6 +197,17 @@ export function verifyRelease(options, run = runGh) {
   }
 }
 
+/** Sync pause used by the post-publication verifier while the release index
+ * catches up with a just-published release.
+ * @param {number} ms
+ */
+function sleepSync(ms) {
+  if (ms <= 0) return
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
+}
+
+const RELEASE_VERIFICATION_ATTEMPTS = 6
+
 /** @param {string[]} argv @param {Runner} [run] */
 export function integrityCommand(argv, run = runGh) {
   const [command, ...args] = argv
@@ -219,17 +239,29 @@ export function integrityCommand(argv, run = runGh) {
     throw new Error('An argument is not supported by this integrity command')
   if (command === 'settings') return verifyImmutableSetting(values['--repo'] ?? '', run)
   if (command === 'manifest') return assetManifest(values['--root'] ?? process.cwd(), assets)
-  if (command === 'release')
-    return verifyRelease(
-      {
-        repository: values['--repo'] ?? '',
-        tag: values['--tag'] ?? '',
-        root: values['--root'],
-        expectedSha: values['--expected-sha'],
-        assets,
-      },
-      run
-    )
+  if (command === 'release') {
+    const options = {
+      repository: values['--repo'] ?? '',
+      tag: values['--tag'] ?? '',
+      root: values['--root'],
+      expectedSha: values['--expected-sha'],
+      assets,
+    }
+    // The lane fires the instant GitHub marks the release published, while the
+    // releases-by-tag index can still lag behind. Retry through that window;
+    // every strict check still runs on every attempt and the command fails
+    // closed once the bounded window is exhausted.
+    let lastError
+    for (let attempt = 1; attempt <= RELEASE_VERIFICATION_ATTEMPTS; attempt += 1) {
+      try {
+        return verifyRelease(options, run)
+      } catch (error) {
+        lastError = error
+        if (attempt < RELEASE_VERIFICATION_ATTEMPTS) sleepSync(5_000)
+      }
+    }
+    throw lastError
+  }
   throw new Error('Use release-integrity settings, release, or manifest')
 }
 

package/src/commands/sync.mjs

@@ -132,6 +132,10 @@ function synchronize(options) {
   if (!['direct', 'staging-release'].includes(workflow)) {
     throw new Error(`Unsupported git_workflow: ${workflow}; use direct or staging-release.`)
   }
+  const draftProtection = configured(config.draft_protection, 'true')
+  if (!['true', 'false'].includes(draftProtection)) {
+    throw new Error(`Unsupported draft_protection: ${draftProtection}; use true or false.`)
+  }
   const obsoleteConfigKeys = [
     'opencode_security',
     ...(workflow === 'direct' ? ['staging_validation_mode'] : []),
@@ -632,13 +636,33 @@ function renderWorkflow(content, config, repository, ref, rustCodeql, file, self
     release: config.release_runner ?? config.runner,
   }
   const workflow = file.match(/^\.github\/workflows\/([^/]+)\.yml$/)?.[1]
+  // Generated PR callers protect drafts by default. Consumers that intentionally
+  // run gates while a PR is still draft can opt out in code-foundry.yml.
+  if (configured(config.draft_protection, 'true') === 'false') {
+    rendered = rendered.replaceAll(' && github.event.pull_request.draft == false', '')
+  }
   // Consumer release callers are generic package release workflows. The
   // installed-consumer qualification harness is specific to Code Foundry's
   // own package and must remain in the self workflow rather than being
   // rendered into every consumer's release caller.
   if (workflow === 'release' && !selfRepository) {
-    rendered = removeWorkflowBlock(rendered, 'qualification')
-    rendered = removeWorkflowNeed(rendered, 'release', 'qualification')
+    // The self caller owns qualification, immutability preflight, draft
+    // recovery, staging, and qualified publication. Consumer callers must
+    // retain the ordinary reusable Release Please path instead of inheriting
+    // self-only jobs whose outputs and environments do not exist for them.
+    for (const job of ['qualification', 'preflight', 'recovery', 'stage', 'publish'])
+      rendered = removeWorkflowBlock(rendered, job)
+    rendered = rendered.replace(/^    needs: \[qualification, preflight\]\n/m, '')
+    rendered = rendered.replace(/^      config-file: \.github\/release-please-foundry\.json\n/m, '')
+    rendered = rendered.replace(/^      defer-publication: true\n/m, '')
+    rendered = rendered.replace(
+      /^    if: github\.ref == 'refs\/heads\/main' && \(vars\.CI_BILLING_PAUSED != 'true' \|\| \(github\.event_name == 'workflow_dispatch' && inputs\['release-while-paused'\] == true\)\)\n/m,
+      "    if: vars.CI_BILLING_PAUSED != 'true' || (github.event_name == 'workflow_dispatch' && inputs['release-while-paused'] == true)\n"
+    )
+    rendered = rendered.replace(
+      /(\n    secrets:\n      CODE_FOUNDRY_TOKEN: \$\{\{ secrets\.CODE_FOUNDRY_TOKEN \}\}\n)/,
+      '$1      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}\n'
+    )
   }
   // The staging-release topology validates and scans pull requests against
   // both main and the integration branch; direct repositories only ever
@@ -690,6 +714,7 @@ function renderWorkflow(content, config, repository, ref, rustCodeql, file, self
       'codeql-runner': config.codeql_runner ?? config.runner,
       'unit-runner': config.unit_runner,
       'performance-runner': config.performance_runner ?? config.test_runner ?? config.runner,
+      'eval-runner': config.eval_runner ?? config.runner,
     }
     for (const [input, value] of Object.entries(runnerInputs)) {
       if (!value) continue
@@ -730,26 +755,6 @@ function removeWorkflowBlock(content, blockId) {
   return lines.join('\n')
 }
 
-/**
- * Remove a single root-job dependency while preserving the rest of the job.
- * @param {string} content
- * @param {string} jobId
- * @param {string} dependency
- * @returns {string}
- */
-function removeWorkflowNeed(content, jobId, dependency) {
-  const lines = content.split('\n')
-  const start = lines.findIndex((line) => line === `  ${jobId}:`)
-  if (start === -1) return content
-  let end = start + 1
-  while (end < lines.length && !/^  [A-Za-z0-9_-]+:\s*$/.test(lines[end])) end += 1
-  const need = lines.findIndex(
-    (line, index) => index > start && index < end && line === `    needs: ${dependency}`
-  )
-  if (need >= 0) lines.splice(need, 1)
-  return lines.join('\n')
-}
-
 /**
  * Dependabot updates land on the repository's integration branch. Direct
  * repositories have no staging branch, so every update targets main. Cargo,
@@ -897,8 +902,8 @@ const DIRECT_DOC_REPLACEMENTS = {
       '| Change                    | Target | Merge method                      | Merge gate                              |\n| ------------------------- | ------ | --------------------------------- | --------------------------------------- |\n| Working branch            | `main` | Squash                            | All applicable required checks pass     |\n| Release Please version PR | `main` | Squash (`release_merge_strategy`) | Validation gate and release policy pass |\n',
     ],
     [
-      'Draft pull requests do not start runner-heavy validation. The lightweight Draft Guard converts ordinary pull requests opened or reopened while ready back to draft; it never checks out pull-request code and it excludes Release Please version heads, whose release workflow owns their state. Marking a pull request ready for review starts the applicable validation tier, and each new commit on a ready pull request reruns that tier for the current head. Draft updates allocate no validation runner. Converting a pull request to draft cancels in-flight validation through the lightweight cancellation control.',
-      'Draft pull requests do not start validation. The lightweight Draft Guard converts ordinary pull requests opened or reopened while ready back to draft; it never checks out pull-request code and it excludes Release Please version heads, whose release workflow owns their state. Marking a pull request ready for review starts the applicable validation tier, and each new commit on a ready pull request reruns that tier for the current head. Draft updates allocate no validation runner. Converting a pull request to draft runs only the lightweight cancellation control.',
+      'Draft pull requests do not start runner-heavy validation unless `draft_protection: false` is configured for generated callers. The lightweight Draft Guard converts ordinary pull requests opened or reopened while ready back to draft; it never checks out pull-request code and it excludes Release Please version heads, whose release workflow owns their state. Marking a pull request ready for review starts the applicable validation tier, and each new commit on a ready pull request reruns that tier for the current head. Draft updates allocate no validation runner while protection is enabled. Converting a pull request to draft cancels in-flight validation through the lightweight cancellation control.',
+      'Draft pull requests do not start validation unless `draft_protection: false` is configured. The lightweight Draft Guard converts ordinary pull requests opened or reopened while ready back to draft; it never checks out pull-request code and it excludes Release Please version heads, whose release workflow owns their state. Marking a pull request ready for review starts the applicable validation tier, and each new commit on a ready pull request reruns that tier for the current head. Draft updates allocate no validation runner while protection is enabled. Converting a pull request to draft runs only the lightweight cancellation control.',
     ],
     ['1. Create a focused branch from `staging`.', '1. Create a focused branch from `main`.'],
   ],
@@ -1370,6 +1375,7 @@ function createDefaultConfig(root, source, configuredWorkflow) {
     toolchain: 'auto',
     ...(stagingRelease ? { staging_validation_mode: 'fast' } : {}),
     performance: 'auto',
+    draft_protection: 'true',
     performance_command: '',
     performance_profile: '',
     performance_budget_file: 'performance-package-budgets.json',

package/src/lib/eval-envelope.mjs

@@ -0,0 +1,234 @@
+// @ts-check
+
+/**
+ * The shared eval-report contract. Every `ci eval` run validates the consumer
+ * repository's eval report against this envelope before applying budgets.
+ * Deterministic probes (Layer 1) and model-agent runs (Layer 2) emit the same
+ * envelope, so task outcomes stay comparable across executors and revisions.
+ * See docs/EVALS.md for the full field reference.
+ */
+
+export const EVAL_REPORT_FILE = 'eval-results/result.json'
+export const EVAL_SUMMARY_FILE = 'eval-results/summary.json'
+export const EVAL_BUDGET_FILE_DEFAULT = 'eval-budgets.json'
+export const EVAL_REPORT_SCHEMA_VERSION = 1
+export const EVAL_SUMMARY_KIND = 'code-foundry-eval-summary'
+
+const FAILURE_CLASSES = ['harness', 'task']
+const STAT_FIELDS = ['mean', 'p50', 'p95', 'max']
+
+/** @param {unknown} value @returns {value is number} */
+function isNonNegativeInteger(value) {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
+/** @param {unknown} value @returns {value is number} */
+function isFiniteNumber(value) {
+  return typeof value === 'number' && Number.isFinite(value)
+}
+
+/** @param {unknown} value @returns {boolean} */
+function isStats(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false
+  const record = /** @type {Record<string, unknown>} */ (value)
+  if (!isNonNegativeInteger(record.count)) return false
+  if (record.count === 0) {
+    return (
+      Object.keys(record).length === 1 || STAT_FIELDS.every((field) => record[field] === undefined)
+    )
+  }
+  return STAT_FIELDS.every((field) => isFiniteNumber(record[field]))
+}
+
+/**
+ * Validate one eval report envelope. Returns every violation instead of
+ * throwing, so one broken report lists all of its problems at once.
+ * @param {unknown} report @returns {{valid: boolean, errors: string[]}}
+ */
+export function validateEvalReport(report) {
+  /** @type {string[]} */
+  const errors = []
+  /** @param {string} message */
+  const push = (message) => errors.push(message)
+  if (!report || typeof report !== 'object' || Array.isArray(report)) {
+    return { valid: false, errors: ['report must be a JSON object'] }
+  }
+  const value = /** @type {Record<string, unknown>} */ (report)
+  if (value.schemaVersion !== EVAL_REPORT_SCHEMA_VERSION)
+    push(`schemaVersion: expected ${EVAL_REPORT_SCHEMA_VERSION}`)
+  if (value.revision !== undefined && typeof value.revision !== 'string')
+    push('revision: must be a string when present')
+  if (value.dependencyHash !== undefined && typeof value.dependencyHash !== 'string')
+    push('dependencyHash: must be a string when present')
+
+  const summary = value.summary
+  if (!summary || typeof summary !== 'object' || Array.isArray(summary)) {
+    push('summary: must be an object')
+    return { valid: errors.length === 0, errors }
+  }
+  const counts = /** @type {Record<string, unknown>} */ (summary)
+  for (const field of [
+    'taskCount',
+    'attempts',
+    'passed',
+    'failed',
+    'harnessFailures',
+    'toolCalls',
+    'evidenceErrors',
+  ]) {
+    if (!isNonNegativeInteger(counts[field]))
+      push(`summary.${field}: must be a non-negative integer`)
+  }
+  if (
+    isNonNegativeInteger(counts.passed) &&
+    isNonNegativeInteger(counts.failed) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.passed + counts.failed > counts.attempts
+  )
+    push('summary.passed/failed: must not exceed attempts')
+  if (
+    isNonNegativeInteger(counts.harnessFailures) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.harnessFailures > counts.attempts
+  )
+    push('summary.harnessFailures: must not exceed attempts')
+  if (isFiniteNumber(counts.successRate) && (counts.successRate < 0 || counts.successRate > 1))
+    push('summary.successRate: must be between 0 and 1')
+  for (const field of ['taskDurationMs', 'startupMs', 'stepDurationMs']) {
+    if (!isStats(counts[field])) push(`summary.${field}: must be a stats object`)
+  }
+
+  const tasks = value.tasks
+  if (!Array.isArray(tasks)) {
+    push('tasks: must be an array')
+    return { valid: errors.length === 0, errors }
+  }
+  for (const [index, task] of tasks.entries()) {
+    if (!task || typeof task !== 'object' || Array.isArray(task)) {
+      push(`tasks[${index}]: must be an object`)
+      continue
+    }
+    const record = /** @type {Record<string, unknown>} */ (task)
+    if (typeof record.id !== 'string' || record.id.trim().length === 0)
+      push(`tasks[${index}].id: must be a non-empty string`)
+    if (!Array.isArray(record.attempts)) {
+      push(`tasks[${index}].attempts: must be an array`)
+      continue
+    }
+    for (const [attemptIndex, attempt] of record.attempts.entries()) {
+      const path = `tasks[${index}].attempts[${attemptIndex}]`
+      if (!attempt || typeof attempt !== 'object') {
+        push(`${path}: must be an object`)
+        continue
+      }
+      const entry = /** @type {Record<string, unknown>} */ (attempt)
+      if (entry.status !== 'passed' && entry.status !== 'failed')
+        push(`${path}.status: must be passed or failed`)
+      if (entry.durationMs !== undefined && !isFiniteNumber(entry.durationMs))
+        push(`${path}.durationMs: must be a finite number`)
+      if (entry.startupMs !== undefined && !isFiniteNumber(entry.startupMs))
+        push(`${path}.startupMs: must be a finite number`)
+      if (entry.failure) {
+        const failure = entry.failure
+        if (!failure || typeof failure !== 'object') push(`${path}.failure: must be an object`)
+        else if (
+          typeof (/** @type {Record<string, unknown>} */ (failure).message) !== 'string' ||
+          /** @type {Record<string, unknown>} */ (failure).message === ''
+        )
+          push(`${path}.failure.message: must be a non-empty string`)
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when a failure is present`)
+      } else if (entry.failureClass !== undefined) {
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when present`)
+      }
+      if (Array.isArray(entry.steps)) {
+        for (const [stepIndex, stepEntry] of entry.steps.entries()) {
+          const stepPath = `${path}.steps[${stepIndex}]`
+          if (!stepEntry || typeof stepEntry !== 'object') {
+            push(`${stepPath}: must be an object`)
+            continue
+          }
+          const step = /** @type {Record<string, unknown>} */ (stepEntry)
+          if (typeof step.tool !== 'string' || step.tool.trim().length === 0)
+            push(`${stepPath}.tool: must be a non-empty string`)
+          if (!['running', 'passed', 'failed'].includes(/** @type {string} */ (step.status)))
+            push(`${stepPath}.status: must be running, passed, or failed`)
+          if (step.durationMs !== undefined && !isFiniteNumber(step.durationMs))
+            push(`${stepPath}.durationMs: must be a finite number`)
+        }
+      }
+    }
+  }
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Evaluate the configured budget thresholds against one validated report.
+ * Unknown budget keys fail closed so a typo can never silently disable a gate.
+ * @param {{summary?: Record<string, unknown>}} report
+ * @param {Record<string, unknown>} budgets
+ * @returns {{passed: boolean, failures: string[]}}
+ */
+export function evaluateEvalBudgets(report, budgets) {
+  if (!report || typeof report !== 'object' || Array.isArray(report))
+    return { passed: false, failures: ['report must be a validated eval report object'] }
+  const failures = []
+  if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) {
+    return { passed: false, failures: ['budgets must be a JSON object'] }
+  }
+  const known = new Set([
+    'successRate',
+    'taskP95Ms',
+    'startupP95Ms',
+    'stepP95Ms',
+    'maxHarnessFailures',
+    'maxEvidenceErrors',
+    'maxToolCalls',
+  ])
+  for (const key of Object.keys(budgets)) {
+    if (!known.has(key)) failures.push(`budgets.${key}: unknown budget key`)
+  }
+  if (failures.length > 0) return { passed: false, failures }
+  /** @type {Record<string, unknown>} */
+  const summary = report.summary ?? {}
+  const rate = budgets.successRate
+  if (rate !== undefined) {
+    if (!isFiniteNumber(rate) || rate < 0 || rate > 1)
+      return { passed: false, failures: ['budgets.successRate: must be between 0 and 1'] }
+    if (isFiniteNumber(summary.successRate) && summary.successRate < rate)
+      failures.push(`successRate ${summary.successRate.toFixed(2)} is below budget ${rate}`)
+  }
+  const p95Fields = /** @type {const} */ ([
+    ['taskP95Ms', 'taskDurationMs'],
+    ['startupP95Ms', 'startupMs'],
+    ['stepP95Ms', 'stepDurationMs'],
+  ])
+  for (const [budgetField, summaryField] of p95Fields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isFiniteNumber(budget) || budget <= 0)
+      return { passed: false, failures: [`budgets.${budgetField}: must be a positive number`] }
+    const stats = summary[summaryField]
+    if (!stats || typeof stats !== 'object') continue
+    const p95 = /** @type {unknown} */ (/** @type {Record<string, unknown>} */ (stats).p95)
+    const count = /** @type {Record<string, unknown>} */ (stats).count
+    if (!isFiniteNumber(p95) || count === 0) continue
+    if (p95 > budget) failures.push(`${summaryField}.p95 ${p95}ms is above budget ${budget}ms`)
+  }
+  const maxFields = /** @type {const} */ ([
+    ['maxHarnessFailures', 'harnessFailures'],
+    ['maxEvidenceErrors', 'evidenceErrors'],
+    ['maxToolCalls', 'toolCalls'],
+  ])
+  for (const [budgetField, summaryField] of maxFields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isNonNegativeInteger(budget))
+      return { passed: false, failures: [`budgets.${budgetField}: must be a non-negative integer`] }
+    const measured = /** @type {unknown} */ (summary[summaryField])
+    if (isNonNegativeInteger(measured) && measured > budget)
+      failures.push(`${summaryField} ${measured} is above budget ${budget}`)
+  }
+  return { passed: failures.length === 0, failures }
+}

package/src/lib/merge-queue.mjs

@@ -153,6 +153,7 @@ jobs:
       unit-runner: ${runner('unit_runner', 'ubuntu-slim')}
       performance-runner: ${runner('performance_runner')}
       security-runner: ${runner('security_runner', 'ubuntu-slim')}
+      eval-runner: ${runner('eval_runner')}
       codeql-runner: ${runner('codeql_runner')}
       rust-shards: '${JSON.stringify(shards)}'
       rust-threads: '${threads}'