code-foundry 1.22.1 → 1.25.3

package/src/commands/qualified-publication.mjs

@@ -46,6 +46,24 @@ export function runCommand(command, args, cwd) {
   )
   return result.stdout
 }
+
+/** Default tolerant runner: GitHub CLI failures surface as retryable misses
+ * instead of aborting the publication flow.
+ * @param {string} command
+ * @param {string[]} args
+ * @param {string} [cwd]
+ * @returns {{status: number, stdout: string}}
+ */
+function defaultRunTolerant(command, args, cwd) {
+  const result = spawnSync(command, args, {
+    cwd,
+    encoding: 'utf8',
+    timeout: 180_000,
+    maxBuffer: 8 * 1024 * 1024,
+    env: { ...process.env, GH_HOST: 'github.com', GH_PROMPT_DISABLED: '1' },
+  })
+  return { status: result.error ? 1 : (result.status ?? 1), stdout: result.stdout ?? '' }
+}
 /** @param {Candidate} candidate */
 export function validateCandidate(candidate) {
   ensure(
@@ -225,10 +243,50 @@ export async function publishQualifiedArchive(candidate, reports, adapters = {})
   return { ...before, status: 'published' }
 }
 
+/** @typedef {(command:string, args:string[], cwd?:string) => {status: number, stdout: string}} RunTolerant */
+
+/** Poll the releases-by-tag endpoint through its eventual-consistency window.
+ * Immediately after `gh release edit --draft=false` the endpoint can still
+ * return 404; a strict verification in that window aborts an otherwise
+ * healthy publication. Bounded retries keep the flow strict: it only proceeds
+ * once the endpoint serves the published release with the exact tag.
+ * @param {Candidate} candidate
+ * @param {{runTolerant?: RunTolerant, delay?: (ms:number)=>Promise<void>, attempts?: number}} [adapters]
+ * @returns {Promise<Record<string, any>>}
+ */
+async function waitForPublishedRelease(candidate, adapters = {}) {
+  const runTolerant = adapters.runTolerant ?? defaultRunTolerant
+  const delay = adapters.delay ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const attempts = adapters.attempts ?? 15
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const result = runTolerant('gh', [
+      'api',
+      '--hostname',
+      'github.com',
+      `repos/${candidate.repository}/releases/tags/${encodeURIComponent(candidate.tag)}`,
+    ])
+    if (result.status === 0) {
+      /** @type {Record<string, any> | null} */
+      let release = null
+      try {
+        release = JSON.parse(result.stdout)
+      } catch {
+        release = null
+      }
+      if (release?.draft === false && release?.tag_name === candidate.tag) return release
+    }
+    if (attempt === attempts) break
+    await delay(2_000)
+  }
+  throw new Error(
+    'The published release was not retrievable via the tag endpoint within the consistency window'
+  )
+}
+
 /** Stage only a previously-created draft: never move tags, overwrite assets, or
  * treat an API/permissions failure as an absent release. Authentication needs
  * immutable-settings read and release write; no setting is mutated.
- * @param {Candidate} candidate @param {string[]} reports @param {{run?:Run, verify?:Verify}} [adapters]
+ * @param {Candidate} candidate @param {string[]} reports @param {{run?:Run, runTolerant?:RunTolerant, delay?:(ms:number)=>Promise<void>, attempts?:number, verify?:Verify}} [adapters]
  */
 export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
   const run = adapters.run ?? runCommand
@@ -332,7 +390,27 @@ export async function stageQualifiedRelease(candidate, reports, adapters = {}) {
       '--repo',
       candidate.repository,
     ])
-    const identity = await (adapters.verify ?? verifier)(candidate)
+    // GitHub's release index is eventually consistent right after a draft is
+    // published; a strict verification a second later can observe a 404 and
+    // abort the publication even though the release is live. Poll until the
+    // index serves the release, then verify; the verification cycle itself
+    // retries through the same window and still fails closed in the end.
+    await waitForPublishedRelease(candidate, {
+      runTolerant: adapters.runTolerant,
+      delay: adapters.delay,
+      attempts: 10,
+    })
+    /** @type {Record<string, any>} */
+    let identity
+    for (let attempt = 1; ; attempt += 1) {
+      try {
+        identity = await (adapters.verify ?? verifier)(candidate)
+        break
+      } catch (error) {
+        if (attempt >= 10) throw error
+        await (adapters.delay ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(5_000)
+      }
+    }
     ensure(
       identity.status === 'passed' &&
         identity.immutable === true &&

package/src/commands/release-integrity.mjs

@@ -7,7 +7,7 @@ import { basename, isAbsolute, relative, resolve, sep } from 'node:path'
 import { spawnSync } from 'node:child_process'
 import { pathToFileURL } from 'node:url'
 
-/** @typedef {(argv: string[]) => {status: number|null, stdout: string}} Runner */
+/** @typedef {(argv: string[]) => {status: number|null, stdout: string, stderr?: string}} Runner */
 
 /** @type {Runner} */
 export function runGh(argv) {
@@ -18,7 +18,7 @@ export function runGh(argv) {
     env: { ...process.env, GH_PROMPT_DISABLED: '1', GH_HOST: 'github.com' },
   })
   if (result.error) throw new Error(`GitHub CLI unavailable: ${result.error.name}`)
-  return { status: result.status, stdout: result.stdout ?? '' }
+  return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }
 }
 
 /** @param {string} repository */
@@ -56,7 +56,7 @@ function json(run, argv) {
   const result = run(argv)
   if (result.status !== 0)
     throw new Error(
-      `GitHub verification command failed (${result.status ?? 'signal'}); state is unverified`
+      `GitHub verification command failed (${result.status ?? 'signal'}); state is unverified: ${String(result.stderr ?? '').slice(0, 300) || 'no stderr captured'}`
     )
   try {
     return JSON.parse(result.stdout)
@@ -197,6 +197,17 @@ export function verifyRelease(options, run = runGh) {
   }
 }
 
+/** Sync pause used by the post-publication verifier while the release index
+ * catches up with a just-published release.
+ * @param {number} ms
+ */
+function sleepSync(ms) {
+  if (ms <= 0) return
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
+}
+
+const RELEASE_VERIFICATION_ATTEMPTS = 6
+
 /** @param {string[]} argv @param {Runner} [run] */
 export function integrityCommand(argv, run = runGh) {
   const [command, ...args] = argv
@@ -228,17 +239,29 @@ export function integrityCommand(argv, run = runGh) {
     throw new Error('An argument is not supported by this integrity command')
   if (command === 'settings') return verifyImmutableSetting(values['--repo'] ?? '', run)
   if (command === 'manifest') return assetManifest(values['--root'] ?? process.cwd(), assets)
-  if (command === 'release')
-    return verifyRelease(
-      {
-        repository: values['--repo'] ?? '',
-        tag: values['--tag'] ?? '',
-        root: values['--root'],
-        expectedSha: values['--expected-sha'],
-        assets,
-      },
-      run
-    )
+  if (command === 'release') {
+    const options = {
+      repository: values['--repo'] ?? '',
+      tag: values['--tag'] ?? '',
+      root: values['--root'],
+      expectedSha: values['--expected-sha'],
+      assets,
+    }
+    // The lane fires the instant GitHub marks the release published, while the
+    // releases-by-tag index can still lag behind. Retry through that window;
+    // every strict check still runs on every attempt and the command fails
+    // closed once the bounded window is exhausted.
+    let lastError
+    for (let attempt = 1; attempt <= RELEASE_VERIFICATION_ATTEMPTS; attempt += 1) {
+      try {
+        return verifyRelease(options, run)
+      } catch (error) {
+        lastError = error
+        if (attempt < RELEASE_VERIFICATION_ATTEMPTS) sleepSync(5_000)
+      }
+    }
+    throw lastError
+  }
   throw new Error('Use release-integrity settings, release, or manifest')
 }
 

package/src/commands/sync.mjs

@@ -714,6 +714,7 @@ function renderWorkflow(content, config, repository, ref, rustCodeql, file, self
       'codeql-runner': config.codeql_runner ?? config.runner,
       'unit-runner': config.unit_runner,
       'performance-runner': config.performance_runner ?? config.test_runner ?? config.runner,
+      'eval-runner': config.eval_runner ?? config.runner,
     }
     for (const [input, value] of Object.entries(runnerInputs)) {
       if (!value) continue

package/src/lib/eval-envelope.mjs

@@ -0,0 +1,234 @@
+// @ts-check
+
+/**
+ * The shared eval-report contract. Every `ci eval` run validates the consumer
+ * repository's eval report against this envelope before applying budgets.
+ * Deterministic probes (Layer 1) and model-agent runs (Layer 2) emit the same
+ * envelope, so task outcomes stay comparable across executors and revisions.
+ * See docs/EVALS.md for the full field reference.
+ */
+
+export const EVAL_REPORT_FILE = 'eval-results/result.json'
+export const EVAL_SUMMARY_FILE = 'eval-results/summary.json'
+export const EVAL_BUDGET_FILE_DEFAULT = 'eval-budgets.json'
+export const EVAL_REPORT_SCHEMA_VERSION = 1
+export const EVAL_SUMMARY_KIND = 'code-foundry-eval-summary'
+
+const FAILURE_CLASSES = ['harness', 'task']
+const STAT_FIELDS = ['mean', 'p50', 'p95', 'max']
+
+/** @param {unknown} value @returns {value is number} */
+function isNonNegativeInteger(value) {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
+/** @param {unknown} value @returns {value is number} */
+function isFiniteNumber(value) {
+  return typeof value === 'number' && Number.isFinite(value)
+}
+
+/** @param {unknown} value @returns {boolean} */
+function isStats(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false
+  const record = /** @type {Record<string, unknown>} */ (value)
+  if (!isNonNegativeInteger(record.count)) return false
+  if (record.count === 0) {
+    return (
+      Object.keys(record).length === 1 || STAT_FIELDS.every((field) => record[field] === undefined)
+    )
+  }
+  return STAT_FIELDS.every((field) => isFiniteNumber(record[field]))
+}
+
+/**
+ * Validate one eval report envelope. Returns every violation instead of
+ * throwing, so one broken report lists all of its problems at once.
+ * @param {unknown} report @returns {{valid: boolean, errors: string[]}}
+ */
+export function validateEvalReport(report) {
+  /** @type {string[]} */
+  const errors = []
+  /** @param {string} message */
+  const push = (message) => errors.push(message)
+  if (!report || typeof report !== 'object' || Array.isArray(report)) {
+    return { valid: false, errors: ['report must be a JSON object'] }
+  }
+  const value = /** @type {Record<string, unknown>} */ (report)
+  if (value.schemaVersion !== EVAL_REPORT_SCHEMA_VERSION)
+    push(`schemaVersion: expected ${EVAL_REPORT_SCHEMA_VERSION}`)
+  if (value.revision !== undefined && typeof value.revision !== 'string')
+    push('revision: must be a string when present')
+  if (value.dependencyHash !== undefined && typeof value.dependencyHash !== 'string')
+    push('dependencyHash: must be a string when present')
+
+  const summary = value.summary
+  if (!summary || typeof summary !== 'object' || Array.isArray(summary)) {
+    push('summary: must be an object')
+    return { valid: errors.length === 0, errors }
+  }
+  const counts = /** @type {Record<string, unknown>} */ (summary)
+  for (const field of [
+    'taskCount',
+    'attempts',
+    'passed',
+    'failed',
+    'harnessFailures',
+    'toolCalls',
+    'evidenceErrors',
+  ]) {
+    if (!isNonNegativeInteger(counts[field]))
+      push(`summary.${field}: must be a non-negative integer`)
+  }
+  if (
+    isNonNegativeInteger(counts.passed) &&
+    isNonNegativeInteger(counts.failed) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.passed + counts.failed > counts.attempts
+  )
+    push('summary.passed/failed: must not exceed attempts')
+  if (
+    isNonNegativeInteger(counts.harnessFailures) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.harnessFailures > counts.attempts
+  )
+    push('summary.harnessFailures: must not exceed attempts')
+  if (isFiniteNumber(counts.successRate) && (counts.successRate < 0 || counts.successRate > 1))
+    push('summary.successRate: must be between 0 and 1')
+  for (const field of ['taskDurationMs', 'startupMs', 'stepDurationMs']) {
+    if (!isStats(counts[field])) push(`summary.${field}: must be a stats object`)
+  }
+
+  const tasks = value.tasks
+  if (!Array.isArray(tasks)) {
+    push('tasks: must be an array')
+    return { valid: errors.length === 0, errors }
+  }
+  for (const [index, task] of tasks.entries()) {
+    if (!task || typeof task !== 'object' || Array.isArray(task)) {
+      push(`tasks[${index}]: must be an object`)
+      continue
+    }
+    const record = /** @type {Record<string, unknown>} */ (task)
+    if (typeof record.id !== 'string' || record.id.trim().length === 0)
+      push(`tasks[${index}].id: must be a non-empty string`)
+    if (!Array.isArray(record.attempts)) {
+      push(`tasks[${index}].attempts: must be an array`)
+      continue
+    }
+    for (const [attemptIndex, attempt] of record.attempts.entries()) {
+      const path = `tasks[${index}].attempts[${attemptIndex}]`
+      if (!attempt || typeof attempt !== 'object') {
+        push(`${path}: must be an object`)
+        continue
+      }
+      const entry = /** @type {Record<string, unknown>} */ (attempt)
+      if (entry.status !== 'passed' && entry.status !== 'failed')
+        push(`${path}.status: must be passed or failed`)
+      if (entry.durationMs !== undefined && !isFiniteNumber(entry.durationMs))
+        push(`${path}.durationMs: must be a finite number`)
+      if (entry.startupMs !== undefined && !isFiniteNumber(entry.startupMs))
+        push(`${path}.startupMs: must be a finite number`)
+      if (entry.failure) {
+        const failure = entry.failure
+        if (!failure || typeof failure !== 'object') push(`${path}.failure: must be an object`)
+        else if (
+          typeof (/** @type {Record<string, unknown>} */ (failure).message) !== 'string' ||
+          /** @type {Record<string, unknown>} */ (failure).message === ''
+        )
+          push(`${path}.failure.message: must be a non-empty string`)
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when a failure is present`)
+      } else if (entry.failureClass !== undefined) {
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when present`)
+      }
+      if (Array.isArray(entry.steps)) {
+        for (const [stepIndex, stepEntry] of entry.steps.entries()) {
+          const stepPath = `${path}.steps[${stepIndex}]`
+          if (!stepEntry || typeof stepEntry !== 'object') {
+            push(`${stepPath}: must be an object`)
+            continue
+          }
+          const step = /** @type {Record<string, unknown>} */ (stepEntry)
+          if (typeof step.tool !== 'string' || step.tool.trim().length === 0)
+            push(`${stepPath}.tool: must be a non-empty string`)
+          if (!['running', 'passed', 'failed'].includes(/** @type {string} */ (step.status)))
+            push(`${stepPath}.status: must be running, passed, or failed`)
+          if (step.durationMs !== undefined && !isFiniteNumber(step.durationMs))
+            push(`${stepPath}.durationMs: must be a finite number`)
+        }
+      }
+    }
+  }
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Evaluate the configured budget thresholds against one validated report.
+ * Unknown budget keys fail closed so a typo can never silently disable a gate.
+ * @param {{summary?: Record<string, unknown>}} report
+ * @param {Record<string, unknown>} budgets
+ * @returns {{passed: boolean, failures: string[]}}
+ */
+export function evaluateEvalBudgets(report, budgets) {
+  if (!report || typeof report !== 'object' || Array.isArray(report))
+    return { passed: false, failures: ['report must be a validated eval report object'] }
+  const failures = []
+  if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) {
+    return { passed: false, failures: ['budgets must be a JSON object'] }
+  }
+  const known = new Set([
+    'successRate',
+    'taskP95Ms',
+    'startupP95Ms',
+    'stepP95Ms',
+    'maxHarnessFailures',
+    'maxEvidenceErrors',
+    'maxToolCalls',
+  ])
+  for (const key of Object.keys(budgets)) {
+    if (!known.has(key)) failures.push(`budgets.${key}: unknown budget key`)
+  }
+  if (failures.length > 0) return { passed: false, failures }
+  /** @type {Record<string, unknown>} */
+  const summary = report.summary ?? {}
+  const rate = budgets.successRate
+  if (rate !== undefined) {
+    if (!isFiniteNumber(rate) || rate < 0 || rate > 1)
+      return { passed: false, failures: ['budgets.successRate: must be between 0 and 1'] }
+    if (isFiniteNumber(summary.successRate) && summary.successRate < rate)
+      failures.push(`successRate ${summary.successRate.toFixed(2)} is below budget ${rate}`)
+  }
+  const p95Fields = /** @type {const} */ ([
+    ['taskP95Ms', 'taskDurationMs'],
+    ['startupP95Ms', 'startupMs'],
+    ['stepP95Ms', 'stepDurationMs'],
+  ])
+  for (const [budgetField, summaryField] of p95Fields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isFiniteNumber(budget) || budget <= 0)
+      return { passed: false, failures: [`budgets.${budgetField}: must be a positive number`] }
+    const stats = summary[summaryField]
+    if (!stats || typeof stats !== 'object') continue
+    const p95 = /** @type {unknown} */ (/** @type {Record<string, unknown>} */ (stats).p95)
+    const count = /** @type {Record<string, unknown>} */ (stats).count
+    if (!isFiniteNumber(p95) || count === 0) continue
+    if (p95 > budget) failures.push(`${summaryField}.p95 ${p95}ms is above budget ${budget}ms`)
+  }
+  const maxFields = /** @type {const} */ ([
+    ['maxHarnessFailures', 'harnessFailures'],
+    ['maxEvidenceErrors', 'evidenceErrors'],
+    ['maxToolCalls', 'toolCalls'],
+  ])
+  for (const [budgetField, summaryField] of maxFields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isNonNegativeInteger(budget))
+      return { passed: false, failures: [`budgets.${budgetField}: must be a non-negative integer`] }
+    const measured = /** @type {unknown} */ (summary[summaryField])
+    if (isNonNegativeInteger(measured) && measured > budget)
+      failures.push(`${summaryField} ${measured} is above budget ${budget}`)
+  }
+  return { passed: failures.length === 0, failures }
+}

package/src/lib/merge-queue.mjs

@@ -153,6 +153,7 @@ jobs:
       unit-runner: ${runner('unit_runner', 'ubuntu-slim')}
       performance-runner: ${runner('performance_runner')}
       security-runner: ${runner('security_runner', 'ubuntu-slim')}
+      eval-runner: ${runner('eval_runner')}
       codeql-runner: ${runner('codeql_runner')}
       rust-shards: '${JSON.stringify(shards)}'
       rust-threads: '${threads}'

package/src/lib/task-policy.mjs

@@ -15,6 +15,7 @@ export const TASKS = Object.freeze([
   'integration',
   'e2e',
   'smoke',
+  'eval',
   'performance',
 ])
 
@@ -28,6 +29,7 @@ export const TASK_SCRIPTS = Object.freeze({
   integration: ['test:integration'],
   e2e: ['test:e2e', 'e2e'],
   smoke: ['test:smoke', 'smoke'],
+  eval: ['eval'],
   performance: ['performance:check', 'perf:check'],
 })
 
@@ -41,6 +43,8 @@ export function readTaskPolicy(root) {
   }
   if (!['true', 'false', 'auto'].includes(config.performance ?? 'auto'))
     throw new Error('performance must be true, false, or auto')
+  if (!['true', 'false', 'auto'].includes(config.eval ?? 'auto'))
+    throw new Error('eval must be true, false, or auto')
   const coverageMode = config.coverage_enforcement ?? 'auto'
   if (!['auto', 'required', 'off'].includes(coverageMode))
     throw new Error('coverage_enforcement must be auto, required, or off')
@@ -50,6 +54,9 @@ export function readTaskPolicy(root) {
     throw new Error('Required performance cannot use performance: false')
   if (config.performance === 'true' && !required.includes('performance'))
     required.push('performance')
+  if (required.includes('eval') && config.eval === 'false')
+    throw new Error('Required eval cannot use eval: false')
+  if (config.eval === 'true' && !required.includes('eval')) required.push('eval')
   const coverageRequired = required.includes('coverage') || coverageMode === 'required'
   if (coverageRequired && !required.includes('unit')) required.push('unit')
   const minimum = Number(config.coverage_minimum ?? '80')

package/src/lib/validation-policy.mjs

@@ -22,7 +22,7 @@ export const AGGREGATE_CHECK_NAME = 'Validation / Gate'
 /** Job ids owned by the validation orchestrator. Release runs the full audit
  * suite and validates its generated diff inside the gate, so no separate
  * release-policy job id exists. */
-export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql']
+export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql', 'eval']
 
 /** Events that may trigger canonical validation. */
 export const VALIDATION_EVENTS = ['pull_request', 'schedule', 'workflow_dispatch']
@@ -81,11 +81,15 @@ export function classifyValidationMode(input) {
 /** @type {Record<'fast'|'audit'|'release', string[]>} */
 const REQUIRED_JOBS_BY_MODE = {
   fast: ['ci', 'test'],
-  audit: ['ci', 'test', 'security', 'codeql'],
-  // Release Please pull requests run the full audit suite so every registered
-  // validation check succeeds rather than appearing as an expected skip. The
-  // gate additionally validates the generated release diff.
-  release: ['ci', 'test', 'security', 'codeql'],
+  audit: ['ci', 'test', 'security', 'codeql', 'eval'],
+  // Release Please pull requests change version metadata only, and the gate
+  // validates that diff against the release policy before anything publishes.
+  // The content tree was already fully audited by the pull requests that
+  // merged into main, and the scheduled audit lane re-covers drift. The
+  // release tier therefore requires the fast suite plus CodeQL, whose
+  // per-pull-request code scanning results some repository rulesets require
+  // to keep the release from deadlocking.
+  release: ['ci', 'test', 'codeql'],
 }
 
 /**