code-foundry 1.20.2 → 1.25.3

package/src/lib/product-quality.mjs

@@ -81,21 +81,226 @@ function decode(value) {
   })
 }
 
+/**
+ * Find an HTML comment terminator, including the parser's comment-end-bang
+ * form. An unterminated comment is treated as extending to the end of input.
+ * @param {string} html
+ * @param {number} start
+ * @returns {{ start: number, length: number } | null}
+ */
+function findCommentEnd(html, start) {
+  const normal = html.indexOf('-->', start + 4)
+  const bang = html.indexOf('--!>', start + 4)
+  if (normal === -1 && bang === -1) return null
+  if (bang !== -1 && (normal === -1 || bang < normal)) return { start: bang, length: 4 }
+  return { start: normal, length: 3 }
+}
+
+/**
+ * Remove comments without leaving an unmatched comment opener that can expose
+ * markup to the lightweight tag reader.
+ * @param {string} html
+ */
+function stripHtmlComments(html) {
+  let clean = ''
+  let cursor = 0
+  while (cursor < html.length) {
+    const start = html.indexOf('<!--', cursor)
+    if (start === -1) return clean + html.slice(cursor)
+    clean += html.slice(cursor, start)
+    const end = findCommentEnd(html, start)
+    if (!end) return clean
+    cursor = end.start + end.length
+  }
+  return clean
+}
+
+/** @param {string} html @param {number} start */
+function findTagEnd(html, start) {
+  let quote = ''
+  for (let index = start + 1; index < html.length; index += 1) {
+    const character = html[index]
+    if (quote) {
+      if (character === quote) quote = ''
+    } else if (character === '"' || character === "'") {
+      quote = character
+    } else if (character === '>') {
+      return index
+    }
+  }
+  return -1
+}
+
+/** @param {string | undefined} character */
+function isTagBoundary(character) {
+  return (
+    character === undefined || character === '>' || character === '/' || character.trim() === ''
+  )
+}
+
+/**
+ * Locate a named HTML tag while accepting parser-tolerated end-tag junk such
+ * as `</script\\t\\n data>`.
+ * @param {string} html
+ * @param {string} name
+ * @param {number} from
+ * @param {boolean} closing
+ * @returns {{ start: number, end: number } | null}
+ */
+function findNamedTag(html, name, from, closing) {
+  const lowerName = name.toLowerCase()
+  for (let start = html.indexOf('<', from); start !== -1; start = html.indexOf('<', start + 1)) {
+    let nameStart = start + 1
+    if (closing) {
+      if (html[nameStart] !== '/') continue
+      nameStart += 1
+    } else if (html[nameStart] === '/' || html[nameStart] === '!' || html[nameStart] === '?') {
+      continue
+    }
+    if (html.slice(nameStart, nameStart + name.length).toLowerCase() !== lowerName) continue
+    if (!isTagBoundary(html[nameStart + name.length])) continue
+    return { start, end: findTagEnd(html, start) }
+  }
+  return null
+}
+
+const RAW_TEXT_TAGS = [
+  'script',
+  'style',
+  'textarea',
+  'template',
+  'xmp',
+  'iframe',
+  'noembed',
+  'noframes',
+  'noscript',
+]
+
+/**
+ * @param {string} html
+ * @param {number} from
+ * @returns {{ start: number, end: number, name: string } | null}
+ */
+function findNextRawTextTag(html, from) {
+  let next = null
+  for (const name of RAW_TEXT_TAGS) {
+    const candidate = findNamedTag(html, name, from, false)
+    if (candidate && (!next || candidate.start < next.start)) {
+      next = { ...candidate, name }
+    }
+  }
+  return next
+}
+
+/**
+ * Remove comments and raw-text contents in document order. Comment markers
+ * inside raw-text elements are data, not document comments.
+ * @param {string} html
+ */
+function stripCommentsAndRawText(html) {
+  let clean = ''
+  let cursor = 0
+  while (cursor < html.length) {
+    const commentStart = html.indexOf('<!--', cursor)
+    const rawText = findNextRawTextTag(html, cursor)
+    if (commentStart === -1 && !rawText) return clean + html.slice(cursor)
+    if (commentStart !== -1 && (!rawText || commentStart < rawText.start)) {
+      clean += html.slice(cursor, commentStart)
+      const commentEnd = findCommentEnd(html, commentStart)
+      if (!commentEnd) return clean
+      cursor = commentEnd.start + commentEnd.length
+      continue
+    }
+    if (!rawText) return clean + html.slice(cursor)
+    clean += html.slice(cursor, rawText.start)
+    if (rawText.end === -1) return clean
+    clean += html.slice(rawText.start, rawText.end + 1)
+    const close = findNamedTag(html, rawText.name, rawText.end + 1, true)
+    if (!close || close.end === -1) return clean
+    cursor = close.end + 1
+  }
+  return clean
+}
+
+/**
+ * Remove angle-bracket markup from title text. A dangling `<` is discarded
+ * through EOF instead of being left as a potentially active tag prefix.
+ * @param {string} value
+ */
+function stripAngleBracketMarkup(value) {
+  let clean = ''
+  let cursor = 0
+  while (cursor < value.length) {
+    const start = value.indexOf('<', cursor)
+    if (start === -1) return clean + value.slice(cursor)
+    clean += value.slice(cursor, start)
+    const end = value.indexOf('>', start + 1)
+    if (end === -1) return clean
+    cursor = end + 1
+  }
+  return clean
+}
+
+/**
+ * Rewrite title bodies without relying on a multi-character HTML-filtering
+ * regexp. Malformed title elements are truncated so their contents cannot be
+ * interpreted as page metadata.
+ * @param {string} html
+ */
+function rewriteTitleMarkup(html) {
+  let clean = ''
+  let cursor = 0
+  while (cursor < html.length) {
+    const open = findNamedTag(html, 'title', cursor, false)
+    if (!open) return clean + html.slice(cursor)
+    clean += html.slice(cursor, open.start)
+    if (open.end === -1) return clean
+    clean += html.slice(open.start, open.end + 1)
+    const close = findNamedTag(html, 'title', open.end + 1, true)
+    if (!close || close.end === -1) return clean
+    clean += stripAngleBracketMarkup(html.slice(open.end + 1, close.start))
+    clean += html.slice(close.start, close.end + 1)
+    cursor = close.end + 1
+  }
+  return clean
+}
+
+/**
+ * Return script bodies while respecting comments and parser-tolerated script
+ * end tags. An unclosed script consumes the remainder of the document.
+ * @param {string} html
+ */
+function readInlineScripts(html) {
+  /** @type {string[]} */
+  const scripts = []
+  let cursor = 0
+  while (cursor < html.length) {
+    const script = findNamedTag(html, 'script', cursor, false)
+    const commentStart = html.indexOf('<!--', cursor)
+    if (commentStart !== -1 && (!script || commentStart < script.start)) {
+      const commentEnd = findCommentEnd(html, commentStart)
+      if (!commentEnd) return scripts
+      cursor = commentEnd.start + commentEnd.length
+      continue
+    }
+    if (!script || script.end === -1) return scripts
+    const contentStart = script.end + 1
+    const candidateClose = findNamedTag(html, 'script', contentStart, true)
+    const close = candidateClose?.end === -1 ? null : candidateClose
+    const contentEnd = close?.start ?? html.length
+    scripts.push(html.slice(contentStart, contentEnd))
+    if (!close || close.end === -1) return scripts
+    cursor = close.end + 1
+  }
+  return scripts
+}
+
 /** Conservative generated-HTML tag reader, not a DOM implementation.
  * Raw-text HTML elements and comments cannot create fake metadata or links.
  * @param {string} html
  */
 export function readTags(html) {
-  const clean = html
-    .replace(/<!--[\s\S]*?-->/g, '')
-    .replace(
-      /(<(script|style|textarea|template|xmp|iframe|noembed|noframes|noscript)\b(?:"[^"]*"|'[^']*'|[^'">])*>)[\s\S]*?<\/\2\s*>/gi,
-      '$1'
-    )
-    .replace(
-      /(<title\b(?:"[^"]*"|'[^']*'|[^'">])*>)([\s\S]*?)<\/title\s*>/gi,
-      (_, open, content) => `${open}${content.replace(/<[^>]*>/g, '')}</title>`
-    )
+  const clean = rewriteTitleMarkup(stripCommentsAndRawText(html))
   /** @type {Array<{ name: string, attrs: Record<string,string> }>} */
   const tags = []
   for (const match of clean.matchAll(/<([a-z][a-z0-9:-]*)\b((?:"[^"]*"|'[^']*'|[^'">])*)>/gi)) {
@@ -245,11 +450,10 @@ export function checkStaticSite(root, profile) {
         requireValue(lstatSync(file).isFile(), 'Budget assets must be regular files')
         return sum + statSync(file).size
       }, 0)
-    const inlineBytes = [
-      ...readFileSync(page.file, 'utf8')
-        .replace(/<!--[\s\S]*?-->/g, '')
-        .matchAll(/<script\b(?:"[^"]*"|'[^']*'|[^'">])*?>([\s\S]*?)<\/script\s*>/gi),
-    ].reduce((sum, match) => sum + Buffer.byteLength(match[1]), 0)
+    const inlineBytes = readInlineScripts(readFileSync(page.file, 'utf8')).reduce(
+      (sum, script) => sum + Buffer.byteLength(script),
+      0
+    )
     const result = {
       route: path,
       htmlBytes: statSync(page.file).size,
@@ -267,7 +471,7 @@ export function checkStaticSite(root, profile) {
   if (profile.sitemap) {
     const sitemap = readFileSync(ownedPath(dist, profile.sitemap), 'utf8')
     requireValue(!/<!DOCTYPE|<!ENTITY/i.test(sitemap), 'Sitemap entities are not supported')
-    const sitemapWithoutComments = sitemap.replace(/<!--[\s\S]*?-->/g, '')
+    const sitemapWithoutComments = stripHtmlComments(sitemap)
     const locations = new Set(
       [
         ...sitemapWithoutComments.matchAll(

package/src/lib/task-policy.mjs

@@ -15,6 +15,7 @@ export const TASKS = Object.freeze([
   'integration',
   'e2e',
   'smoke',
+  'eval',
   'performance',
 ])
 
@@ -28,6 +29,7 @@ export const TASK_SCRIPTS = Object.freeze({
   integration: ['test:integration'],
   e2e: ['test:e2e', 'e2e'],
   smoke: ['test:smoke', 'smoke'],
+  eval: ['eval'],
   performance: ['performance:check', 'perf:check'],
 })
 
@@ -41,6 +43,8 @@ export function readTaskPolicy(root) {
   }
   if (!['true', 'false', 'auto'].includes(config.performance ?? 'auto'))
     throw new Error('performance must be true, false, or auto')
+  if (!['true', 'false', 'auto'].includes(config.eval ?? 'auto'))
+    throw new Error('eval must be true, false, or auto')
   const coverageMode = config.coverage_enforcement ?? 'auto'
   if (!['auto', 'required', 'off'].includes(coverageMode))
     throw new Error('coverage_enforcement must be auto, required, or off')
@@ -50,6 +54,9 @@ export function readTaskPolicy(root) {
     throw new Error('Required performance cannot use performance: false')
   if (config.performance === 'true' && !required.includes('performance'))
     required.push('performance')
+  if (required.includes('eval') && config.eval === 'false')
+    throw new Error('Required eval cannot use eval: false')
+  if (config.eval === 'true' && !required.includes('eval')) required.push('eval')
   const coverageRequired = required.includes('coverage') || coverageMode === 'required'
   if (coverageRequired && !required.includes('unit')) required.push('unit')
   const minimum = Number(config.coverage_minimum ?? '80')

package/src/lib/validation-policy.mjs

@@ -22,7 +22,7 @@ export const AGGREGATE_CHECK_NAME = 'Validation / Gate'
 /** Job ids owned by the validation orchestrator. Release runs the full audit
  * suite and validates its generated diff inside the gate, so no separate
  * release-policy job id exists. */
-export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql']
+export const VALIDATION_JOBS = ['ci', 'test', 'security', 'codeql', 'eval']
 
 /** Events that may trigger canonical validation. */
 export const VALIDATION_EVENTS = ['pull_request', 'schedule', 'workflow_dispatch']
@@ -81,11 +81,15 @@ export function classifyValidationMode(input) {
 /** @type {Record<'fast'|'audit'|'release', string[]>} */
 const REQUIRED_JOBS_BY_MODE = {
   fast: ['ci', 'test'],
-  audit: ['ci', 'test', 'security', 'codeql'],
-  // Release Please pull requests run the full audit suite so every registered
-  // validation check succeeds rather than appearing as an expected skip. The
-  // gate additionally validates the generated release diff.
-  release: ['ci', 'test', 'security', 'codeql'],
+  audit: ['ci', 'test', 'security', 'codeql', 'eval'],
+  // Release Please pull requests change version metadata only, and the gate
+  // validates that diff against the release policy before anything publishes.
+  // The content tree was already fully audited by the pull requests that
+  // merged into main, and the scheduled audit lane re-covers drift. The
+  // release tier therefore requires the fast suite plus CodeQL, whose
+  // per-pull-request code scanning results some repository rulesets require
+  // to keep the release from deadlocking.
+  release: ['ci', 'test', 'codeql'],
 }
 
 /**

package/src/runtime-core.mjs

@@ -10,6 +10,13 @@ import { classifyTestFiles } from './lib/test-discovery.mjs'
 import { classifyValidationMode, evaluateValidationGate } from './lib/validation-policy.mjs'
 import { readReleaseConfig, validateGeneratedReleaseDiff } from './lib/release-policy.mjs'
 import { runNodePackagePerformance } from './lib/node-package-performance.mjs'
+import {
+  EVAL_BUDGET_FILE_DEFAULT,
+  EVAL_REPORT_FILE,
+  EVAL_SUMMARY_FILE,
+  evaluateEvalBudgets,
+  validateEvalReport,
+} from './lib/eval-envelope.mjs'
 
 const root = process.cwd()
 const config = readConfig(resolve(root, '.github/code-foundry.yml'))
@@ -57,6 +64,26 @@ function performanceEnabled() {
   return configured(config.performance, 'auto') !== 'false'
 }
 
+function evalEnabled() {
+  return configured(config.eval, 'auto') !== 'false'
+}
+
+function evalCommand() {
+  const raw = configured(config.eval_command, '').trim()
+  if (!raw) return null
+  let command
+  try {
+    command = JSON.parse(raw)
+  } catch {
+    throw new Error('eval_command must be a JSON argv array.')
+  }
+  if (!Array.isArray(command) || command.length === 0)
+    throw new Error('eval_command must be a non-empty JSON array.')
+  if (!command.every((argument) => typeof argument === 'string' && argument.length > 0))
+    throw new Error('eval_command must contain only non-empty strings.')
+  return /** @type {string[]} */ (command)
+}
+
 function performanceCommands() {
   const raw = configured(config.performance_command, '').trim()
   if (!raw) return []
@@ -98,7 +125,8 @@ function selectedPerformanceCommands() {
   const name = ['performance:check', 'perf:check'].find((candidate) => hasScript(candidate))
   if (name) {
     const [manager, args] = packageCommand(['run', name])
-    return manager ? [{ source: `package-script:${name}`, argv: [manager, ...args] }] : []
+    if (!manager) throw new Error(`Cannot run ${name}: select a supported package_manager.`)
+    return [{ source: `package-script:${name}`, argv: [manager, ...args] }]
   }
   return performanceCommands().map((argv) => ({ source: 'configuration', argv }))
 }
@@ -127,6 +155,122 @@ function writePerformanceSummary(startedAt, status, commands, artifacts, error =
   )
 }
 
+const evalResultsDirectory = 'eval-results'
+
+/** @returns {{source: string, argv: string[]}[]} */
+function selectedEvalCommands() {
+  const name = ['eval'].find((candidate) => hasScript(candidate))
+  if (name) {
+    const [manager, args] = packageCommand(['run', name])
+    if (!manager) throw new Error(`Cannot run ${name}: select a supported package_manager.`)
+    return [{ source: `package-script:${name}`, argv: [manager, ...args] }]
+  }
+  const command = evalCommand()
+  if (!command) throw new Error('No eval script or eval_command was discovered.')
+  return [{ source: 'configuration', argv: command }]
+}
+
+/** @param {string} startedAt @param {'passed'|'failed'} status @param {{source: string, argv: string[], status: number}[]} commands @param {string[]} artifacts @param {string|null} error @param {{file: string, applied: boolean, failures: string[]}|null} budgets */
+function writeEvalSummary(startedAt, status, commands, artifacts, error = null, budgets = null) {
+  const directory = resolve(root, evalResultsDirectory)
+  mkdirSync(directory, { recursive: true })
+  writeFileSync(
+    resolve(directory, 'summary.json'),
+    `${JSON.stringify(
+      {
+        schemaVersion: 1,
+        kind: 'code-foundry-eval-summary',
+        status,
+        startedAt,
+        completedAt: new Date().toISOString(),
+        commands,
+        report: configured(config.eval_report_file, EVAL_REPORT_FILE),
+        budgets,
+        artifacts,
+        error,
+      },
+      null,
+      2
+    )}\n`
+  )
+}
+
+function runEval() {
+  if (!evalEnabled()) return
+  const startedAt = new Date().toISOString()
+  /** @type {{source: string, argv: string[], status: number}[]} */
+  const records = []
+  const artifacts = [EVAL_REPORT_FILE, EVAL_SUMMARY_FILE]
+  const budgetFile = configured(config.eval_budget_file, EVAL_BUDGET_FILE_DEFAULT)
+  /** @type {{file: string, applied: boolean, failures: string[]}|null} */
+  let budgets = null
+  try {
+    for (const command of selectedEvalCommands()) {
+      const result = spawnSync(command.argv[0], command.argv.slice(1), {
+        cwd: root,
+        stdio: 'inherit',
+        env: process.env,
+      })
+      if (result.error) throw result.error
+      const status = result.status ?? 1
+      records.push({ ...command, status })
+      if (status !== 0) {
+        writeEvalSummary(
+          startedAt,
+          'failed',
+          records,
+          artifacts,
+          `command exited ${status}`,
+          budgets
+        )
+        process.exitCode = status
+        return
+      }
+    }
+    const reportFile = resolve(root, configured(config.eval_report_file, EVAL_REPORT_FILE))
+    if (!existsSync(reportFile))
+      throw new Error(
+        `Eval report was not produced: ${configured(config.eval_report_file, EVAL_REPORT_FILE)}`
+      )
+    let report
+    try {
+      report = JSON.parse(readFileSync(reportFile, 'utf8'))
+    } catch (error) {
+      throw new Error(
+        `Eval report is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
+      )
+    }
+    const envelope = validateEvalReport(report)
+    if (!envelope.valid)
+      throw new Error(`Eval report violates the contract: ${envelope.errors.join('; ')}`)
+    if (existsSync(resolve(root, budgetFile))) {
+      const raw = JSON.parse(readFileSync(resolve(root, budgetFile), 'utf8'))
+      const gate = evaluateEvalBudgets(report, raw)
+      budgets = { file: budgetFile, applied: true, failures: gate.failures }
+      if (!gate.passed) {
+        for (const failure of gate.failures) console.error(`::error::${failure}`)
+        writeEvalSummary(
+          startedAt,
+          'failed',
+          records,
+          artifacts,
+          `eval budgets failed: ${gate.failures.join('; ')}`,
+          budgets
+        )
+        process.exitCode = 1
+        return
+      }
+    } else {
+      budgets = { file: budgetFile, applied: false, failures: [] }
+    }
+    writeEvalSummary(startedAt, 'passed', records, artifacts, null, budgets)
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    writeEvalSummary(startedAt, 'failed', records, artifacts, message, budgets)
+    throw error
+  }
+}
+
 function runPerformance() {
   if (!performanceEnabled()) return
   const startedAt = new Date().toISOString()
@@ -194,6 +338,7 @@ function validation(task) {
         test: process.env.FOUNDRY_TEST,
         security: process.env.FOUNDRY_SECURITY,
         codeql: process.env.FOUNDRY_CODEQL,
+        eval: process.env.FOUNDRY_EVAL,
       },
     })
     if (gate.valid) {
@@ -390,8 +535,15 @@ function relevant(task) {
     integration: ['test:integration'],
     e2e: ['test:e2e', 'e2e'],
     smoke: ['test:smoke', 'smoke'],
+    eval: ['eval'],
     performance: ['performance:check', 'perf:check'],
   }[task]
+  if (task === 'eval') {
+    if (!evalEnabled()) return false
+    return Boolean(
+      (scripted && scripted.some((candidate) => hasScript(candidate))) || evalCommand()
+    )
+  }
   if (task === 'performance') {
     if (!performanceEnabled()) return false
     return Boolean(
@@ -562,6 +714,9 @@ function ci(task) {
       run('cargo', ['build', '--all-targets'])
     return
   }
+  if (task === 'eval') {
+    return runEval()
+  }
   if (task === 'performance') {
     return runPerformance()
   }
@@ -600,15 +755,18 @@ function ci(task) {
 
   if (hasLanguage('rust') && hasRootRustProject()) {
     if (task === 'unit') {
-      if (existsSync(resolve(root, 'src/lib.rs'))) run('cargo', ['test', '--lib'])
-      if (existsSync(resolve(root, 'src/main.rs'))) run('cargo', ['test', '--bin', packageName()])
-      if (!existsSync(resolve(root, 'src/lib.rs')) && !existsSync(resolve(root, 'src/main.rs')))
-        run('cargo', ['test'])
+      // Batch exact prior targets through one Cargo graph.
+      const args = ['test']
+      if (existsSync(resolve(root, 'src/lib.rs'))) args.push('--lib')
+      if (existsSync(resolve(root, 'src/main.rs'))) args.push('--bin', packageName())
+      run('cargo', args)
     } else {
-      for (const file of rustTests) {
+      // Keep --test selection narrow; --tests includes unit targets.
+      const targets = rustTests.flatMap((file) => {
         const match = file.match(/^tests\/(.+)\.rs$/)
-        if (match && !match[1].includes('/')) run('cargo', ['test', '--test', match[1]])
-      }
+        return match && !match[1].includes('/') ? ['--test', match[1]] : []
+      })
+      if (targets.length > 0) run('cargo', ['test', ...targets])
     }
   }
 }

package/src/runtime.mjs

@@ -81,7 +81,7 @@ function sourceSha(root) {
 }
 
 /**
- * Stable public runtime; the original ecosystem executor is kept private and unchanged.
+ * Stable public runtime; the native executor stays separate from policy and evidence.
  * Tests can substitute an executor without installing consumer dependencies.
  * @param {string[]} args @param {string} [root] @param {string} [entry]
  * @returns {number}
@@ -186,6 +186,8 @@ export function runRuntime(args, root = process.cwd(), entry = core) {
       report.coverage = evaluateCoverage(root, policy, before)
       report.artifacts.push(...report.coverage.artifacts)
     }
+    if (task === 'eval')
+      report.artifacts.push('eval-results/summary.json', 'eval-results/result.json')
     if (task === 'performance') report.artifacts.push('performance-results/summary.json')
     report.status = 'passed'
     return 0

package/src/templates/gitignore

@@ -41,12 +41,14 @@ htmlcov/
 artifacts/
 performance-results.json
 performance-results/
+eval-results.json
+eval-results/
 cache/
 !.github/actions/cache/
 !.github/actions/cache/action.yml
 typechain/
 coverage.json
-performance-results/
+.code-foundry/
 tests/.bin/
 **/.app/