code-foundry 1.22.1 → 1.28.2

package/src/lib/eval-envelope.mjs

@@ -0,0 +1,234 @@
+// @ts-check
+
+/**
+ * The shared eval-report contract. Every `ci eval` run validates the consumer
+ * repository's eval report against this envelope before applying budgets.
+ * Deterministic probes (Layer 1) and model-agent runs (Layer 2) emit the same
+ * envelope, so task outcomes stay comparable across executors and revisions.
+ * See docs/EVALS.md for the full field reference.
+ */
+
+export const EVAL_REPORT_FILE = 'eval-results/result.json'
+export const EVAL_SUMMARY_FILE = 'eval-results/summary.json'
+export const EVAL_BUDGET_FILE_DEFAULT = 'eval-budgets.json'
+export const EVAL_REPORT_SCHEMA_VERSION = 1
+export const EVAL_SUMMARY_KIND = 'code-foundry-eval-summary'
+
+const FAILURE_CLASSES = ['harness', 'task']
+const STAT_FIELDS = ['mean', 'p50', 'p95', 'max']
+
+/** @param {unknown} value @returns {value is number} */
+function isNonNegativeInteger(value) {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
+/** @param {unknown} value @returns {value is number} */
+function isFiniteNumber(value) {
+  return typeof value === 'number' && Number.isFinite(value)
+}
+
+/** @param {unknown} value @returns {boolean} */
+function isStats(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false
+  const record = /** @type {Record<string, unknown>} */ (value)
+  if (!isNonNegativeInteger(record.count)) return false
+  if (record.count === 0) {
+    return (
+      Object.keys(record).length === 1 || STAT_FIELDS.every((field) => record[field] === undefined)
+    )
+  }
+  return STAT_FIELDS.every((field) => isFiniteNumber(record[field]))
+}
+
+/**
+ * Validate one eval report envelope. Returns every violation instead of
+ * throwing, so one broken report lists all of its problems at once.
+ * @param {unknown} report @returns {{valid: boolean, errors: string[]}}
+ */
+export function validateEvalReport(report) {
+  /** @type {string[]} */
+  const errors = []
+  /** @param {string} message */
+  const push = (message) => errors.push(message)
+  if (!report || typeof report !== 'object' || Array.isArray(report)) {
+    return { valid: false, errors: ['report must be a JSON object'] }
+  }
+  const value = /** @type {Record<string, unknown>} */ (report)
+  if (value.schemaVersion !== EVAL_REPORT_SCHEMA_VERSION)
+    push(`schemaVersion: expected ${EVAL_REPORT_SCHEMA_VERSION}`)
+  if (value.revision !== undefined && typeof value.revision !== 'string')
+    push('revision: must be a string when present')
+  if (value.dependencyHash !== undefined && typeof value.dependencyHash !== 'string')
+    push('dependencyHash: must be a string when present')
+
+  const summary = value.summary
+  if (!summary || typeof summary !== 'object' || Array.isArray(summary)) {
+    push('summary: must be an object')
+    return { valid: errors.length === 0, errors }
+  }
+  const counts = /** @type {Record<string, unknown>} */ (summary)
+  for (const field of [
+    'taskCount',
+    'attempts',
+    'passed',
+    'failed',
+    'harnessFailures',
+    'toolCalls',
+    'evidenceErrors',
+  ]) {
+    if (!isNonNegativeInteger(counts[field]))
+      push(`summary.${field}: must be a non-negative integer`)
+  }
+  if (
+    isNonNegativeInteger(counts.passed) &&
+    isNonNegativeInteger(counts.failed) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.passed + counts.failed > counts.attempts
+  )
+    push('summary.passed/failed: must not exceed attempts')
+  if (
+    isNonNegativeInteger(counts.harnessFailures) &&
+    isNonNegativeInteger(counts.attempts) &&
+    counts.harnessFailures > counts.attempts
+  )
+    push('summary.harnessFailures: must not exceed attempts')
+  if (isFiniteNumber(counts.successRate) && (counts.successRate < 0 || counts.successRate > 1))
+    push('summary.successRate: must be between 0 and 1')
+  for (const field of ['taskDurationMs', 'startupMs', 'stepDurationMs']) {
+    if (!isStats(counts[field])) push(`summary.${field}: must be a stats object`)
+  }
+
+  const tasks = value.tasks
+  if (!Array.isArray(tasks)) {
+    push('tasks: must be an array')
+    return { valid: errors.length === 0, errors }
+  }
+  for (const [index, task] of tasks.entries()) {
+    if (!task || typeof task !== 'object' || Array.isArray(task)) {
+      push(`tasks[${index}]: must be an object`)
+      continue
+    }
+    const record = /** @type {Record<string, unknown>} */ (task)
+    if (typeof record.id !== 'string' || record.id.trim().length === 0)
+      push(`tasks[${index}].id: must be a non-empty string`)
+    if (!Array.isArray(record.attempts)) {
+      push(`tasks[${index}].attempts: must be an array`)
+      continue
+    }
+    for (const [attemptIndex, attempt] of record.attempts.entries()) {
+      const path = `tasks[${index}].attempts[${attemptIndex}]`
+      if (!attempt || typeof attempt !== 'object') {
+        push(`${path}: must be an object`)
+        continue
+      }
+      const entry = /** @type {Record<string, unknown>} */ (attempt)
+      if (entry.status !== 'passed' && entry.status !== 'failed')
+        push(`${path}.status: must be passed or failed`)
+      if (entry.durationMs !== undefined && !isFiniteNumber(entry.durationMs))
+        push(`${path}.durationMs: must be a finite number`)
+      if (entry.startupMs !== undefined && !isFiniteNumber(entry.startupMs))
+        push(`${path}.startupMs: must be a finite number`)
+      if (entry.failure) {
+        const failure = entry.failure
+        if (!failure || typeof failure !== 'object') push(`${path}.failure: must be an object`)
+        else if (
+          typeof (/** @type {Record<string, unknown>} */ (failure).message) !== 'string' ||
+          /** @type {Record<string, unknown>} */ (failure).message === ''
+        )
+          push(`${path}.failure.message: must be a non-empty string`)
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when a failure is present`)
+      } else if (entry.failureClass !== undefined) {
+        if (!FAILURE_CLASSES.includes(/** @type {string} */ (entry.failureClass)))
+          push(`${path}.failureClass: must be harness or task when present`)
+      }
+      if (Array.isArray(entry.steps)) {
+        for (const [stepIndex, stepEntry] of entry.steps.entries()) {
+          const stepPath = `${path}.steps[${stepIndex}]`
+          if (!stepEntry || typeof stepEntry !== 'object') {
+            push(`${stepPath}: must be an object`)
+            continue
+          }
+          const step = /** @type {Record<string, unknown>} */ (stepEntry)
+          if (typeof step.tool !== 'string' || step.tool.trim().length === 0)
+            push(`${stepPath}.tool: must be a non-empty string`)
+          if (!['running', 'passed', 'failed'].includes(/** @type {string} */ (step.status)))
+            push(`${stepPath}.status: must be running, passed, or failed`)
+          if (step.durationMs !== undefined && !isFiniteNumber(step.durationMs))
+            push(`${stepPath}.durationMs: must be a finite number`)
+        }
+      }
+    }
+  }
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Evaluate the configured budget thresholds against one validated report.
+ * Unknown budget keys fail closed so a typo can never silently disable a gate.
+ * @param {{summary?: Record<string, unknown>}} report
+ * @param {Record<string, unknown>} budgets
+ * @returns {{passed: boolean, failures: string[]}}
+ */
+export function evaluateEvalBudgets(report, budgets) {
+  if (!report || typeof report !== 'object' || Array.isArray(report))
+    return { passed: false, failures: ['report must be a validated eval report object'] }
+  const failures = []
+  if (!budgets || typeof budgets !== 'object' || Array.isArray(budgets)) {
+    return { passed: false, failures: ['budgets must be a JSON object'] }
+  }
+  const known = new Set([
+    'successRate',
+    'taskP95Ms',
+    'startupP95Ms',
+    'stepP95Ms',
+    'maxHarnessFailures',
+    'maxEvidenceErrors',
+    'maxToolCalls',
+  ])
+  for (const key of Object.keys(budgets)) {
+    if (!known.has(key)) failures.push(`budgets.${key}: unknown budget key`)
+  }
+  if (failures.length > 0) return { passed: false, failures }
+  /** @type {Record<string, unknown>} */
+  const summary = report.summary ?? {}
+  const rate = budgets.successRate
+  if (rate !== undefined) {
+    if (!isFiniteNumber(rate) || rate < 0 || rate > 1)
+      return { passed: false, failures: ['budgets.successRate: must be between 0 and 1'] }
+    if (isFiniteNumber(summary.successRate) && summary.successRate < rate)
+      failures.push(`successRate ${summary.successRate.toFixed(2)} is below budget ${rate}`)
+  }
+  const p95Fields = /** @type {const} */ ([
+    ['taskP95Ms', 'taskDurationMs'],
+    ['startupP95Ms', 'startupMs'],
+    ['stepP95Ms', 'stepDurationMs'],
+  ])
+  for (const [budgetField, summaryField] of p95Fields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isFiniteNumber(budget) || budget <= 0)
+      return { passed: false, failures: [`budgets.${budgetField}: must be a positive number`] }
+    const stats = summary[summaryField]
+    if (!stats || typeof stats !== 'object') continue
+    const p95 = /** @type {unknown} */ (/** @type {Record<string, unknown>} */ (stats).p95)
+    const count = /** @type {Record<string, unknown>} */ (stats).count
+    if (!isFiniteNumber(p95) || count === 0) continue
+    if (p95 > budget) failures.push(`${summaryField}.p95 ${p95}ms is above budget ${budget}ms`)
+  }
+  const maxFields = /** @type {const} */ ([
+    ['maxHarnessFailures', 'harnessFailures'],
+    ['maxEvidenceErrors', 'evidenceErrors'],
+    ['maxToolCalls', 'toolCalls'],
+  ])
+  for (const [budgetField, summaryField] of maxFields) {
+    const budget = /** @type {unknown} */ (budgets[budgetField])
+    if (budget === undefined) continue
+    if (!isNonNegativeInteger(budget))
+      return { passed: false, failures: [`budgets.${budgetField}: must be a non-negative integer`] }
+    const measured = /** @type {unknown} */ (summary[summaryField])
+    if (isNonNegativeInteger(measured) && measured > budget)
+      failures.push(`${summaryField} ${measured} is above budget ${budget}`)
+  }
+  return { passed: failures.length === 0, failures }
+}

package/src/lib/fleet-manifest.mjs

@@ -277,16 +277,20 @@ export function verifiedPullRequest(pr, required) {
     return false
   const checks = pr.statusCheckRollup
   if (!Array.isArray(checks) || !checks.length) return false
-  /** @param {any} check */
-  const state = (check) =>
-    check.status === 'COMPLETED' ? check.conclusion : (check.state ?? check.status)
-  if (checks.some((check) => !['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(state(check))))
+  if (checks.some((check) => !['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(checkState(check))))
     return false
   return required.every((name) =>
-    checks.some((check) => (check.name ?? check.context) === name && state(check) === 'SUCCESS')
+    checks.some(
+      (check) => (check.name ?? check.context) === name && checkState(check) === 'SUCCESS'
+    )
   )
 }
 
+/** @param {any} check */
+function checkState(check) {
+  return check.status === 'COMPLETED' ? check.conclusion : (check.state ?? check.status)
+}
+
 /** @param {string} text */
 function scalarConfig(text) {
   return Object.fromEntries(

package/src/lib/merge-queue.mjs

@@ -153,6 +153,7 @@ jobs:
       unit-runner: ${runner('unit_runner', 'ubuntu-slim')}
       performance-runner: ${runner('performance_runner')}
       security-runner: ${runner('security_runner', 'ubuntu-slim')}
+      eval-runner: ${runner('eval_runner')}
       codeql-runner: ${runner('codeql_runner')}
       rust-shards: '${JSON.stringify(shards)}'
       rust-threads: '${threads}'

package/src/lib/overlay.mjs

@@ -16,7 +16,7 @@ export function customWorkflowFiles(root, standardFiles) {
     .filter((file) => file.endsWith('.yml') || file.endsWith('.yaml'))
     .map((file) => `.github/workflows/${file}`)
     .filter((file) => !isManagedPath(standardFiles, file))
-    .sort()
+    .toSorted()
 }
 
 /** @param {string} root @param {Record<string, string>} config */

package/src/lib/release-manifest.mjs

@@ -61,7 +61,7 @@ export function detectReleasePackages(root) {
       })
     }
   })
-  return packages.sort((a, b) => a.directory.localeCompare(b.directory))
+  return packages.toSorted((a, b) => a.directory.localeCompare(b.directory))
 }
 
 /**

package/src/lib/release-policy.mjs

@@ -295,8 +295,8 @@ export function buildReleaseRecoveryPlan(input) {
   const orphanGitHubReleases = releaseTags.filter(
     (tag) => !tagSet.has(tag) && !tagSet.has(tag.replace(/^v/, ''))
   )
-  const latestTag = [...tags].sort(compareVersions).at(-1) ?? ''
-  const latestPackageVersion = [...input.packageVersions].sort(compareVersions).at(-1) ?? ''
+  const latestTag = [...tags].toSorted(compareVersions).at(-1) ?? ''
+  const latestPackageVersion = [...input.packageVersions].toSorted(compareVersions).at(-1) ?? ''
   return {
     latestTag,
     latestPackageVersion,

package/src/lib/task-policy.mjs

@@ -15,6 +15,7 @@ export const TASKS = Object.freeze([
   'integration',
   'e2e',
   'smoke',
+  'eval',
   'performance',
 ])
 
@@ -28,6 +29,7 @@ export const TASK_SCRIPTS = Object.freeze({
   integration: ['test:integration'],
   e2e: ['test:e2e', 'e2e'],
   smoke: ['test:smoke', 'smoke'],
+  eval: ['eval'],
   performance: ['performance:check', 'perf:check'],
 })
 
@@ -41,6 +43,8 @@ export function readTaskPolicy(root) {
   }
   if (!['true', 'false', 'auto'].includes(config.performance ?? 'auto'))
     throw new Error('performance must be true, false, or auto')
+  if (!['true', 'false', 'auto'].includes(config.eval ?? 'auto'))
+    throw new Error('eval must be true, false, or auto')
   const coverageMode = config.coverage_enforcement ?? 'auto'
   if (!['auto', 'required', 'off'].includes(coverageMode))
     throw new Error('coverage_enforcement must be auto, required, or off')
@@ -50,6 +54,9 @@ export function readTaskPolicy(root) {
     throw new Error('Required performance cannot use performance: false')
   if (config.performance === 'true' && !required.includes('performance'))
     required.push('performance')
+  if (required.includes('eval') && config.eval === 'false')
+    throw new Error('Required eval cannot use eval: false')
+  if (config.eval === 'true' && !required.includes('eval')) required.push('eval')
   const coverageRequired = required.includes('coverage') || coverageMode === 'required'
   if (coverageRequired && !required.includes('unit')) required.push('unit')
   const minimum = Number(config.coverage_minimum ?? '80')

package/src/lib/validation-policy.mjs

@@ -22,7 +22,7 @@ export const AGGREGATE_CHECK_NAME = 'Validation / Gate'
 /** Job ids owned by the validation orchestrator. Release runs the full audit
  * suite and validates its generated diff inside the gate, so no separate
  * release-policy job id exists. */
-export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql']
+export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql', 'eval']
 
 /** Events that may trigger canonical validation. */
 export const VALIDATION_EVENTS = ['pull_request', 'schedule', 'workflow_dispatch']
@@ -81,11 +81,15 @@ export function classifyValidationMode(input) {
 /** @type {Record<'fast'|'audit'|'release', string[]>} */
 const REQUIRED_JOBS_BY_MODE = {
   fast: ['ci', 'test'],
-  audit: ['ci', 'test', 'security', 'codeql'],
-  // Release Please pull requests run the full audit suite so every registered
-  // validation check succeeds rather than appearing as an expected skip. The
-  // gate additionally validates the generated release diff.
-  release: ['ci', 'test', 'security', 'codeql'],
+  audit: ['ci', 'test', 'security', 'codeql', 'eval'],
+  // Release Please pull requests change version metadata only, and the gate
+  // validates that diff against the release policy before anything publishes.
+  // The content tree was already fully audited by the pull requests that
+  // merged into main, and the scheduled audit lane re-covers drift. The
+  // release tier therefore requires the fast suite plus CodeQL, whose
+  // per-pull-request code scanning results some repository rulesets require
+  // to keep the release from deadlocking.
+  release: ['ci', 'test', 'codeql'],
 }
 
 /**

package/src/runtime-core.mjs

@@ -7,9 +7,17 @@ import { spawnSync } from 'node:child_process'
 import { detectPackageManager, resolveProfile } from './lib/profile.mjs'
 import { configured, readConfig } from './lib/config.mjs'
 import { classifyTestFiles } from './lib/test-discovery.mjs'
+import { docsOnlyPullRequest } from './lib/docs-only.mjs'
 import { classifyValidationMode, evaluateValidationGate } from './lib/validation-policy.mjs'
 import { readReleaseConfig, validateGeneratedReleaseDiff } from './lib/release-policy.mjs'
 import { runNodePackagePerformance } from './lib/node-package-performance.mjs'
+import {
+  EVAL_BUDGET_FILE_DEFAULT,
+  EVAL_REPORT_FILE,
+  EVAL_SUMMARY_FILE,
+  evaluateEvalBudgets,
+  validateEvalReport,
+} from './lib/eval-envelope.mjs'
 
 const root = process.cwd()
 const config = readConfig(resolve(root, '.github/code-foundry.yml'))
@@ -57,6 +65,26 @@ function performanceEnabled() {
   return configured(config.performance, 'auto') !== 'false'
 }
 
+function evalEnabled() {
+  return configured(config.eval, 'auto') !== 'false'
+}
+
+function evalCommand() {
+  const raw = configured(config.eval_command, '').trim()
+  if (!raw) return null
+  let command
+  try {
+    command = JSON.parse(raw)
+  } catch {
+    throw new Error('eval_command must be a JSON argv array.')
+  }
+  if (!Array.isArray(command) || command.length === 0)
+    throw new Error('eval_command must be a non-empty JSON array.')
+  if (!command.every((argument) => typeof argument === 'string' && argument.length > 0))
+    throw new Error('eval_command must contain only non-empty strings.')
+  return /** @type {string[]} */ (command)
+}
+
 function performanceCommands() {
   const raw = configured(config.performance_command, '').trim()
   if (!raw) return []
@@ -128,6 +156,123 @@ function writePerformanceSummary(startedAt, status, commands, artifacts, error =
   )
 }
 
+const evalResultsDirectory = 'eval-results'
+
+/** @returns {{source: string, argv: string[]}[]} */
+function selectedEvalCommands() {
+  const name = ['eval'].find((candidate) => hasScript(candidate))
+  if (name) {
+    const [manager, args] = packageCommand(['run', name])
+    if (!manager) throw new Error(`Cannot run ${name}: select a supported package_manager.`)
+    return [{ source: `package-script:${name}`, argv: [manager, ...args] }]
+  }
+  const command = evalCommand()
+  if (!command) throw new Error('No eval script or eval_command was discovered.')
+  return [{ source: 'configuration', argv: command }]
+}
+
+/** @param {string} startedAt @param {'passed'|'failed'} status @param {{source: string, argv: string[], status: number}[]} commands @param {string[]} artifacts @param {string|null} error @param {{file: string, applied: boolean, failures: string[]}|null} budgets */
+function writeEvalSummary(startedAt, status, commands, artifacts, error = null, budgets = null) {
+  const directory = resolve(root, evalResultsDirectory)
+  mkdirSync(directory, { recursive: true })
+  writeFileSync(
+    resolve(directory, 'summary.json'),
+    `${JSON.stringify(
+      {
+        schemaVersion: 1,
+        kind: 'code-foundry-eval-summary',
+        status,
+        startedAt,
+        completedAt: new Date().toISOString(),
+        commands,
+        report: configured(config.eval_report_file, EVAL_REPORT_FILE),
+        budgets,
+        artifacts,
+        error,
+      },
+      null,
+      2
+    )}\n`
+  )
+}
+
+function runEval() {
+  if (!evalEnabled()) return
+  const startedAt = new Date().toISOString()
+  /** @type {{source: string, argv: string[], status: number}[]} */
+  const records = []
+  const artifacts = [EVAL_REPORT_FILE, EVAL_SUMMARY_FILE]
+  const budgetFile = configured(config.eval_budget_file, EVAL_BUDGET_FILE_DEFAULT)
+  /** @type {{file: string, applied: boolean, failures: string[]}|null} */
+  let budgets = null
+  try {
+    for (const command of selectedEvalCommands()) {
+      const result = spawnSync(command.argv[0], command.argv.slice(1), {
+        cwd: root,
+        stdio: 'inherit',
+        env: process.env,
+      })
+      if (result.error) throw result.error
+      const status = result.status ?? 1
+      records.push({ ...command, status })
+      if (status !== 0) {
+        writeEvalSummary(
+          startedAt,
+          'failed',
+          records,
+          artifacts,
+          `command exited ${status}`,
+          budgets
+        )
+        process.exitCode = status
+        return
+      }
+    }
+    const reportFile = resolve(root, configured(config.eval_report_file, EVAL_REPORT_FILE))
+    if (!existsSync(reportFile))
+      throw new Error(
+        `Eval report was not produced: ${configured(config.eval_report_file, EVAL_REPORT_FILE)}`
+      )
+    let report
+    try {
+      report = JSON.parse(readFileSync(reportFile, 'utf8'))
+    } catch (error) {
+      throw new Error(
+        `Eval report is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error }
+      )
+    }
+    const envelope = validateEvalReport(report)
+    if (!envelope.valid)
+      throw new Error(`Eval report violates the contract: ${envelope.errors.join('; ')}`)
+    if (existsSync(resolve(root, budgetFile))) {
+      const raw = JSON.parse(readFileSync(resolve(root, budgetFile), 'utf8'))
+      const gate = evaluateEvalBudgets(report, raw)
+      budgets = { file: budgetFile, applied: true, failures: gate.failures }
+      if (!gate.passed) {
+        for (const failure of gate.failures) console.error(`::error::${failure}`)
+        writeEvalSummary(
+          startedAt,
+          'failed',
+          records,
+          artifacts,
+          `eval budgets failed: ${gate.failures.join('; ')}`,
+          budgets
+        )
+        process.exitCode = 1
+        return
+      }
+    } else {
+      budgets = { file: budgetFile, applied: false, failures: [] }
+    }
+    writeEvalSummary(startedAt, 'passed', records, artifacts, null, budgets)
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    writeEvalSummary(startedAt, 'failed', records, artifacts, message, budgets)
+    throw error
+  }
+}
+
 function runPerformance() {
   if (!performanceEnabled()) return
   const startedAt = new Date().toISOString()
@@ -172,17 +317,37 @@ function runPerformance() {
   }
 }
 
+/**
+ * Audit-mode pull requests whose diff is docs-only run the fast tier instead.
+ * Scheduled and manual audits always keep the full tier; release pull requests
+ * are classified separately and always run their lane.
+ * @param {string} mode
+ */
+function downgradeDocsOnlyPullRequest(mode) {
+  if (mode !== 'audit') return mode
+  if ((process.env.FOUNDRY_EVENT_NAME ?? '') !== 'pull_request') return mode
+  const baseRef = process.env.FOUNDRY_BASE_REF ?? ''
+  if (!baseRef) return mode
+  try {
+    return docsOnlyPullRequest(root, baseRef) ? 'fast' : mode
+  } catch {
+    return mode
+  }
+}
+
 /** @param {string} task */
 function validation(task) {
   if (task === 'mode') {
     writeOutput(
       'mode',
-      classifyValidationMode({
-        eventName: process.env.FOUNDRY_EVENT_NAME ?? '',
-        baseRef: process.env.FOUNDRY_BASE_REF ?? '',
-        headRef: process.env.FOUNDRY_HEAD_REF ?? '',
-        stagingMode: config.staging_validation_mode,
-      })
+      downgradeDocsOnlyPullRequest(
+        classifyValidationMode({
+          eventName: process.env.FOUNDRY_EVENT_NAME ?? '',
+          baseRef: process.env.FOUNDRY_BASE_REF ?? '',
+          headRef: process.env.FOUNDRY_HEAD_REF ?? '',
+          stagingMode: config.staging_validation_mode,
+        })
+      )
     )
     return
   }
@@ -195,6 +360,7 @@ function validation(task) {
         test: process.env.FOUNDRY_TEST,
         security: process.env.FOUNDRY_SECURITY,
         codeql: process.env.FOUNDRY_CODEQL,
+        eval: process.env.FOUNDRY_EVAL,
       },
     })
     if (gate.valid) {
@@ -391,8 +557,15 @@ function relevant(task) {
     integration: ['test:integration'],
     e2e: ['test:e2e', 'e2e'],
     smoke: ['test:smoke', 'smoke'],
+    eval: ['eval'],
     performance: ['performance:check', 'perf:check'],
   }[task]
+  if (task === 'eval') {
+    if (!evalEnabled()) return false
+    return Boolean(
+      (scripted && scripted.some((candidate) => hasScript(candidate))) || evalCommand()
+    )
+  }
   if (task === 'performance') {
     if (!performanceEnabled()) return false
     return Boolean(
@@ -541,10 +714,11 @@ function ci(task) {
       hasRootJavascriptProject() &&
       hasOxlintSetup()
     ) {
-      // Oxlint is the baseline's linter. Repositories that use a different
-      // linter keep full control through their own `lint` script, which the
-      // runScript fallback above already honors.
-      runTool('oxlint', [])
+      // Oxlint is the baseline's linter. Warnings are denied so lint debt
+      // never accumulates silently. Repositories that use a different linter
+      // keep full control through their own `lint` script, which the runScript
+      // fallback above already honors.
+      runTool('oxlint', ['--deny-warnings'])
     }
     if (hasLanguage('python') && hasRootPythonProject()) runTool('ruff', ['check', '.'])
     if (hasLanguage('rust') && hasRootRustProject())
@@ -563,6 +737,9 @@ function ci(task) {
       run('cargo', ['build', '--all-targets'])
     return
   }
+  if (task === 'eval') {
+    return runEval()
+  }
   if (task === 'performance') {
     return runPerformance()
   }

package/src/runtime.mjs

@@ -186,6 +186,8 @@ export function runRuntime(args, root = process.cwd(), entry = core) {
       report.coverage = evaluateCoverage(root, policy, before)
       report.artifacts.push(...report.coverage.artifacts)
     }
+    if (task === 'eval')
+      report.artifacts.push('eval-results/summary.json', 'eval-results/result.json')
     if (task === 'performance') report.artifacts.push('performance-results/summary.json')
     report.status = 'passed'
     return 0

package/src/templates/gitignore

@@ -41,6 +41,8 @@ htmlcov/
 artifacts/
 performance-results.json
 performance-results/
+eval-results.json
+eval-results/
 cache/
 !.github/actions/cache/
 !.github/actions/cache/action.yml