kaloko 0.18.0 → 0.19.0

package/src/lib/evaluate.mjs

@@ -1,7 +1,9 @@
 import { join, resolve } from 'node:path'
 import { readJson, writeJson, sha256 } from './util.mjs'
 import { readCaptures, runPaths } from './run.mjs'
-import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
+import { buildRequest, verdictFor, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
+import { textAsker, evaluatorBudget, jevSettings, safeUrl, LOCAL_PROMPT_VERSION } from './local-evaluator.mjs'
+import { appliesToState, STACK_SEMANTIC_SAMPLE } from './manifest-core.mjs'
 import { visionSettings, visionCriteria, templateReason, VISION_PROMPT_VERSION } from './vision-core.mjs'
 import { prepareImages, visionPlan, askVision, hostedVisionSender } from './vision.mjs'
 import { triageSettings, probeFindings, TRIAGE_PROMPT_VERSION } from './triage-core.mjs'
@@ -10,7 +12,7 @@ import { KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
 import { packHelp } from './pack-help.mjs'
 import { tokenCriteria, storedValueChecks, STYLE_ASSERTS } from './assertions.mjs'
-import { appliesToState } from './manifest-core.mjs'
+import { errorPageOnly, isErrorPage } from './packs.mjs'
 import { isHealthCriterion, findingSeverity, toAcceptanceId } from './health-core.mjs'
 
 /** Latest capture per node/locale/viewport. */
@@ -28,6 +30,8 @@ export const criteriaHash = (node) => sha256((node.criteria || []).map(({ accept
 export const appliesOn = (c, environment) => !environment || !Array.isArray(c.environments) || !c.environments.length || c.environments.includes(environment)
 /** Environment and step state (#46): a criterion with `states:` applies only to captures of those states. */
 export const appliesTo = (c, environment, capture) => appliesOn(c, environment) && appliesToState(c, capture?.step_state)
+/** The errors pack's soft-404 and explanation checks (HC601, HC605) apply to error pages only (packs.mjs isErrorPage). */
+export const appliesToPage = (c, node, capture) => !errorPageOnly(c) || isErrorPage(node, capture)
 /** Folder of a capture under steps/: the node, or steps/<node>/items/<item> for a page of a stack step. */
 export const stepIdOf = (e) => (e.item ? `${e.node}/items/${e.item}` : e.node)
 export const evaluationSource = (node, capture) => sha256({ capture, criteria: node.criteria || [], purpose: node.purpose || '' })
@@ -41,10 +45,11 @@ export function combine ({ node, capture, jev, vision = null, triage = null, cas
   return (node.criteria || []).map((c) => {
     if (!appliesOn(c, environment)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies on ${c.environments.join(', ')} only; this walk ran on ${environment}.`, na: true, not_applicable: 'environment' }
     if (!appliesToState(c, capture?.step_state)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies to the state${c.states.length > 1 ? 's' : ''} ${c.states.join(', ')} only; this capture shows ${capture?.step_state || 'default'}.`, na: true, not_applicable: 'state' }
+    if (!appliesToPage(c, node, capture)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Checked on error pages only; this page answered HTTP ${capture?.http_status ?? '(unknown)'} and the step declares no expect_status of 400 or more.`, na: true, not_applicable: 'error_page' }
     if (c.check.type === 'deterministic') {
       const r = capture.checks?.[c.id]
       const verdict = r == null || r.ok == null ? 'not_evaluated' : r.ok ? 'pass' : 'fail'
-      const live = c.check.assert === 'no_console_errors' || c.check.assert === 'no_overflow' || c.check.assert === 'http_status' || c.check.assert === 'links_ok' || c.check.assert === 'hreflang_pairs' || /^perf_/.test(c.check.assert || '') || KEYBOARD_ASSERTS.has(c.check.assert)
+      const live = c.check.assert === 'no_console_errors' || c.check.assert === 'no_overflow' || c.check.assert === 'http_status' || c.check.assert === 'links_ok' || c.check.assert === 'hreflang_pairs' || c.check.assert === 'llms_txt' || c.check.assert === 'nojs_parity' || /^perf_/.test(c.check.assert || '') || KEYBOARD_ASSERTS.has(c.check.assert)
       return { id: c.id, type: 'deterministic', verdict, evidence: r?.evidence ?? null, ...(r?.na ? { na: true } : {}), ...(capture.rechecked_at && !live ? { rechecked: true } : {}) }
     }
     if (c.check.type === 'semantic' && c.check.about === 'console' && !(capture.console_errors || []).length && capture.checks) {
@@ -53,6 +58,12 @@ export function combine ({ node, capture, jev, vision = null, triage = null, cas
     if (c.check.type === 'semantic') {
       const answer = jev?.response?.answers?.[c.id]
       const text = answer?.noul ?? null
+      // the text evaluator that gave this answer: JEV, your own model, or JEV after your model was unsure (#158)
+      const own = jev?.response?.provider === 'openai-compatible'
+      const esc = jev?.response?.escalation
+      const textProvider = answer?.escalated ? esc?.provider : jev?.response?.provider || 'typesafe-jev'
+      // with your own model in play every text verdict names who decided it: via, model and prompt
+      const origin = !own || !answer ? null : answer.escalated ? { via: esc.via, model: esc.model, prompt_version: esc.prompt_version } : { via: 'own-model', model: jev.response.model, prompt_version: LOCAL_PROMPT_VERSION }
       const seen = vision?.answers?.[c.id] || null
       // Clef's answer on the screenshot (#71): recorded beside the others; with cascade on it decides when it is sure
       const clef = triage?.criteria?.[c.id] ?? null
@@ -62,10 +73,12 @@ export function combine ({ node, capture, jev, vision = null, triage = null, cas
       const reason = seen?.reason || (clefDecides ? `Judged from the screenshot by Clef: ${Math.round(clef * 100)} % yes, clear enough for a verdict.` : templateReason({ score, scope: c.check.scope, about: c.check.about, verdict }))
       return {
         id: c.id, type: 'semantic', verdict, evidence: null, score,
-        provider: seen ? `${vision.provider || 'anthropic'}-vision` : clefDecides ? 'cloudflare-clef' : answer ? 'typesafe-jev' : null,
-        note: score == null ? (jev?.skipped || jev?.error || vision?.skipped || vision?.error || null) : null,
+        provider: seen ? `${vision.provider || 'anthropic'}-vision` : clefDecides ? 'cloudflare-clef' : answer ? textProvider : null,
+        note: score == null ? (jev?.skipped || jev?.error || (own ? esc?.skipped || esc?.error || 'your own evaluator gave no usable answer' : null) || vision?.skipped || vision?.error || null) : null,
+        ...(origin && !seen && !clefDecides ? origin : {}),
         ...(reason ? { reason, reason_source: seen?.reason ? 'model' : 'template' } : {}),
         ...((seen || clefDecides) && text != null ? { text_score: text } : {}),
+        ...(answer?.escalated && answer.first_noul != null ? { first_score: answer.first_noul } : {}),
         ...(clef != null ? { clef_score: clef } : {}),
         ...(!seen && (vision?.skipped || vision?.error) && vision.ids?.includes(c.id) ? { vision_note: vision.skipped || vision.error } : {})
       }
@@ -76,14 +89,77 @@ export function combine ({ node, capture, jev, vision = null, triage = null, cas
 
 const BATCH = 50
 
-export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
-  const settings = { ...DEFAULTS, ...(config.evaluators?.jev || {}), thresholds: { ...DEFAULTS.thresholds, ...(config.evaluators?.jev?.thresholds || {}) } }
-  const cacheDir = join(resolve(root, config.output), '.cache', 'jev')
+/**
+ * What decides a stack page's place in a prioritised sample: a failing deterministic acceptance criterion of the step
+ * (advisory health checks do not count), and when the page was checked last.
+ */
+export const itemFacts = (capture, node = null) => {
+  const acceptance = node ? new Set((node.criteria || []).filter((c) => c.check?.type === 'deterministic' && !isHealthCriterion(c)).map((c) => c.id)) : null
+  return {
+    failing: Object.entries(capture?.checks || {}).some(([id, r]) => r && r.ok === false && !r.na && (!acceptance || acceptance.has(id))),
+    last_checked: capture?.item?.last_checked ?? null
+  }
+}
+
+/** A stack step's sample is chosen by priority with a budget or items.semantic_order: priority; else in walk order. */
+export const prioritised = (node, budget) => node?.items?.semantic_order === 'priority' || (budget?.limit != null && node?.items?.semantic_order !== 'walk')
+
+/**
+ * The pages of each stack step that get the semantic questions (items.semantic_sample, default 20). With facts for a
+ * step (a prioritised sample, #158) the most useful come first: pages failing a deterministic check, then pages never
+ * or least recently checked (last_checked of a rotating sample), then the rest in walk order; without, walk order.
+ * Stable: the same captures give the same sample.
+ * facts: Map "node|item" → { failing, last_checked } (itemFacts of each capture, merged). Returns Map node → item ids
+ * in that order.
+ */
+export function sampleItems (latest, nodes, facts = new Map()) {
+  const order = new Map()
+  for (const e of latest) {
+    if (!e.item || !nodes.has(e.node)) continue
+    const list = order.get(e.node) || []; order.set(e.node, list)
+    if (!list.includes(e.item)) list.push(e.item)
+  }
+  const out = new Map()
+  for (const [node, items] of order) {
+    const n = nodes.get(node).items?.semantic_sample ?? STACK_SEMANTIC_SAMPLE
+    const ranked = items.map((id, at) => ({ id, at, ...(facts.get(`${node}|${id}`) || { failing: false, last_checked: null }) }))
+      .sort((a, b) => (b.failing - a.failing) || ((a.last_checked != null) - (b.last_checked != null)) || String(a.last_checked ?? '').localeCompare(String(b.last_checked ?? '')) || a.at - b.at)
+    out.set(node, ranked.slice(0, n).map((r) => r.id))
+  }
+  return out
+}
+
+/** Facts of the pages of prioritised stacks whose sample is a choice (more pages than semantic_sample). */
+async function stackFacts (latest, nodes, runDir, budget) {
+  const count = new Map()
+  for (const e of latest) if (e.item) { const s = count.get(e.node) || new Set(); s.add(e.item); count.set(e.node, s) }
+  const facts = new Map()
+  for (const e of latest) {
+    if (!e.item) continue
+    const node = nodes.get(e.node)
+    if (!node || !prioritised(node, budget) || count.get(e.node).size <= (node.items?.semantic_sample ?? STACK_SEMANTIC_SAMPLE)) continue
+    const f = itemFacts(await readJson(join(runDir, e.file), null), node)
+    const k = `${e.node}|${e.item}`; const was = facts.get(k)
+    facts.set(k, was ? { failing: was.failing || f.failing, last_checked: was.last_checked == null || f.last_checked == null ? null : was.last_checked < f.last_checked ? was.last_checked : f.last_checked } : f)
+  }
+  return facts
+}
+
+/** Evaluates the latest captures of a run. `budget` (kaloko evaluate --budget) overrides evaluate.budget.calls. */
+export async function evaluateRun ({ run, scenario, config, root, fetchImpl, budget: budgetCalls = null }) {
+  const settings = jevSettings(config)
   const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
   const paths = runPaths(run.dir)
   // criteria limited to other environments (environments: [production]) are reported as not applicable here
   const environment = run.meta?.environment?.name || null
   const report = { evaluated: 0, semantic_calls: 0, semantic_cached: 0, skipped: [], model: settings.model, drift: 0, outdated: new Set(), via: {}, hosted: null, hostedStop: null, findings: [] }
+  // per-evaluation budget of evaluator calls (#158) and the text evaluator: JEV, or your own model escalating to it
+  const budget = evaluatorBudget(config, budgetCalls)
+  let budgetSkipped = 0 // captures whose semantic criteria the budget left out
+  const hosted = hostedSender(config, report)
+  const text = textAsker({ config, root, fetchImpl, hosted, budget, stopped: () => report.hostedStop })
+  report.model = text.model
+  if (text.primary.kind !== 'jev') report.evaluator = { name: text.primary.name, base_url: safeUrl(text.primary.settings.base_url), ...(text.escalate ? { escalate_to: text.escalate.name, escalated: 0, escalation_skipped: 0 } : {}) }
   // the vision evaluator (#17): flagged and uncertain criteria, a bounded number of calls per evaluation
   const vset = visionSettings(config)
   if (vset) report.vision = { model: vset.model, asked: 0, calls: 0, cached: 0, skipped: [], via: {}, hosted: null }
@@ -101,17 +177,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   try {
   const entries = await readCaptures(run.dir)
   const latest = latestCaptures(entries)
-  // stack steps: semantic criteria only on the first N pages (items.semantic_sample, default 20); cost stays bounded
-  const sampled = new Map()
-  for (const e of latest) {
-    if (!e.item) continue
-    const node = nodes.get(e.node); if (!node) continue
-    const n = node.items?.semantic_sample ?? 20
-    const set = sampled.get(e.node) || new Set(); sampled.set(e.node, set)
-    if (set.size < n) set.add(e.item)
+  // stack steps: semantic criteria only on N pages (items.semantic_sample, default 20); cost stays bounded. With a
+  // budget or semantic_order: priority, failing and least recently checked pages first
+  const chosen = sampleItems(latest, nodes, await stackFacts(latest, nodes, run.dir, budget))
+  const sampled = new Map([...chosen].map(([node, ids]) => [node, new Set(ids)]))
+  // with a budget the calls go where they matter most: steps first, then each stack's sample in its order
+  if (budget.limit != null) {
+    const rank = new Map([...chosen].flatMap(([node, ids]) => ids.map((id, i) => [`${node}|${id}`, i])))
+    const key = (e) => (!e.item ? -1 : rank.get(`${e.node}|${e.item}`) ?? Infinity)
+    latest.sort((a, b) => key(a) - key(b))
   }
   // in batches of 50 captures: memory holds one batch of page states, not a whole stack of thousands
-  const hosted = hostedSender(config, report)
   for (let at = 0; at < latest.length; at += BATCH) {
     const batch = latest.slice(at, at + BATCH)
     // 1. what each capture needs; 2. the evaluator calls, several at a time; 3. drift and verdicts in order
@@ -125,25 +201,28 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
       const current = capture.criteria_hash === criteriaHash(node)
       // console triage needs the evaluator only when the capture has console errors
       const outOfSample = entry.item && !sampled.get(entry.node)?.has(entry.item)
-      const semantic = outOfSample ? [] : (node.criteria || []).filter((c) => c.check.type === 'semantic' && appliesTo(c, environment, capture) && !(c.check.about === 'console' && !(capture.console_errors || []).length))
+      const semantic = outOfSample ? [] : (node.criteria || []).filter((c) => c.check.type === 'semantic' && appliesTo(c, environment, capture) && appliesToPage(c, node, capture) && !(c.check.about === 'console' && !(capture.console_errors || []).length))
       // quick signals ride along on every captured page state (also steps without semantic criteria)
       const signals = settings.signals !== false && !outOfSample && !!capture.state
       // an imported screen without page text (#11) has nothing to ask about: a person decides on the canvas
       const textless = capture.capture?.source === 'import' && !capture.state
-      const request = (semantic.length || signals) && current && !textless ? buildRequest({ node, capture, criteria: semantic, model: settings.model, signals }) : null
+      const request = (semantic.length || signals) && current && !textless ? buildRequest({ node, capture, criteria: semantic, model: text.model, signals }) : null
       work.push({ entry, node, capture, current, outOfSample, request, asks: semantic.length > 0, jev: !current ? { skipped: 'Capture has no matching criteria fingerprint; capture this step again.' } : textless && semantic.length ? { skipped: 'imported screenshot without page text; a person decides on the canvas' } : null })
       if (!current) report.outdated.add(entry.node)
     }
 
+    // the text evaluator answers the batch in its order (budget tickets follow it, not who answers first)
     const queue = work.filter((w) => w.request)
-    await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 6, queue.length)) }, async () => {
-      for (let w = queue.shift(); w; w = queue.shift()) {
-        w.jev = report.hostedStop && !process.env[settings.api_key_env] ? { skipped: report.hostedStop } : await evaluateWithJev({ request: w.request, settings, cacheDir, fetchImpl, hosted })
-        if (w.jev.response) { w.jev.cached ? report.semantic_cached++ : report.semantic_calls++; report.via[w.jev.via] = (report.via[w.jev.via] || 0) + 1; if (w.jev.response.qawalk) report.hosted = w.jev.response.qawalk }
-        // signals alone are best effort: only captures with semantic criteria report why they were not evaluated
-        if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
-      }
-    }))
+    const answers = await text.askAll(queue.map((w) => ({ request: w.request, asks: w.asks })))
+    for (const [i, w] of queue.entries()) {
+      w.jev = answers[i]
+      if (w.jev.response) { w.jev.cached ? report.semantic_cached++ : report.semantic_calls++; report.via[w.jev.via] = (report.via[w.jev.via] || 0) + 1; if (w.jev.response.qawalk) report.hosted = w.jev.response.qawalk }
+      const esc = w.jev.response?.escalation
+      if (esc && report.evaluator) { if (esc.via) { report.evaluator.escalated += esc.ids.length; report.via[esc.via] = (report.via[esc.via] || 0) + 1 } else report.evaluator.escalation_skipped += esc.ids.length }
+      if (w.jev.budget && w.asks) budgetSkipped++
+      // signals alone are best effort: only captures with semantic criteria report why they were not evaluated
+      if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
+    }
     // what kind of problem each screen and its recording shows; with cascade on, Clef may settle uncertain criteria
     if (triage) await triage.ask(work.filter((w) => w.current && !w.outOfSample), run.dir, entries)
     // second opinion with the screenshot for flagged and uncertain criteria (those Clef did not settle)
@@ -161,15 +240,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         node: entry.node, ...(entry.item ? { item: entry.item, semantic_sample: !outOfSample } : {}), locale: entry.locale, viewport: entry.viewport,
         evaluated_at: new Date().toISOString(),
         semantic: jev?.response ? {
-          provider: 'typesafe-jev',
+          provider: jev.response.provider || 'typesafe-jev',
           via: jev.via || 'own-key',
-          model: jev.response.model || settings.model,
-          prompt_version: PROMPT_VERSION,
+          model: jev.response.model || text.model,
+          prompt_version: jev.response.provider === 'openai-compatible' ? LOCAL_PROMPT_VERSION : PROMPT_VERSION,
           state_hash: jev.key,
           cached: jev.cached,
           usage: jev.response.usage || null,
-          answers: jev.response.answers
-        } : (jev ? { provider: 'typesafe-jev', skipped: jev.skipped || null, error: jev.error || null } : null),
+          answers: jev.response.answers,
+          // uncertain answers re-asked of the escalation evaluator (#158)
+          ...(jev.response.escalation ? { escalation: jev.response.escalation } : {})
+        } : (jev ? { provider: text.primary.kind === 'jev' ? 'typesafe-jev' : 'openai-compatible', skipped: jev.skipped || null, error: jev.error || null, ...(jev.budget ? { budget: true } : {}) } : null),
         thresholds: settings.thresholds,
         state_chars: request ? JSON.stringify(request.state).length : null,
         ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
@@ -189,6 +270,7 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   // triage is on by default but stays silent where nobody asked for it and it did not run (experiment off, no service)
   if (report.triage && !report.triage.screens && config.evaluators?.triage == null) delete report.triage
   if (report.triage) { const ms = report.triage.ms.filter((x) => x != null).sort((a, b) => a - b); report.triage.median_ms = ms.length ? ms[Math.floor(ms.length / 2)] : null; delete report.triage.ms }
+  if (budget.limit != null) report.budget = { calls: budget.limit, used: budget.used, skipped: budgetSkipped }
   const { hostedStop, visionStop, triageStop, ...out } = report
   return { ...out, outdated: [...report.outdated] }
 }
@@ -207,16 +289,19 @@ function signalsOf (jev) {
   return Object.keys(out).length ? out : null
 }
 
-export function semanticPrefetch ({ scenario, config, root, environment = null }) {
-  const settings = { ...DEFAULTS, ...(config.evaluators?.jev || {}), thresholds: { ...DEFAULTS.thresholds, ...(config.evaluators?.jev?.thresholds || {}) } }
-  const cacheDir = join(resolve(root, config.output), '.cache', 'jev')
+export function semanticPrefetch ({ scenario, config, root, environment = null, budget: budgetCalls = null, fetchImpl }) {
+  const settings = jevSettings(config)
   const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
   const report = { hostedStop: null }
   const hosted = hostedSender(config, report)
+  // with a budget the paid questions wait for the evaluation at the end of the walk, which asks them in a fixed
+  // order; the walk asks only your own model (free), without escalating
+  const budgeted = evaluatorBudget(config, budgetCalls).limit != null
+  const text = textAsker({ config, root, fetchImpl, hosted, budget: null, stopped: () => report.hostedStop })
   const queue = []; const running = new Set(); let asked = 0
   let idle = null // resolves when the queue drains, for done()
   const pump = () => {
-    while (running.size < (settings.concurrency || 6) && queue.length) {
+    while (running.size < text.concurrency && queue.length) {
       const job = queue.shift()
       const p = job().catch(() => {}).finally(() => { running.delete(p); pump() })
       running.add(p)
@@ -226,6 +311,7 @@ export function semanticPrefetch ({ scenario, config, root, environment = null }
   return {
     add (entries, runDir) {
       if (config.evaluators?.jev === false || config.evaluators?.jev === null) return
+      if (budgeted && text.primary.kind === 'jev') return
       for (const e of [].concat(entries || [])) {
         if (e?.type !== 'capture' || e.item) continue
         const node = nodes.get(e.node)
@@ -233,11 +319,11 @@ export function semanticPrefetch ({ scenario, config, root, environment = null }
         if (!semantic.length && settings.signals === false) continue
         queue.push(async () => {
           const capture = await readJson(join(runDir, e.file))
-          const crit = semantic.filter((c) => appliesToState(c, capture.step_state) && !(c.check.about === 'console' && !(capture.console_errors || []).length))
+          const crit = semantic.filter((c) => appliesToState(c, capture.step_state) && appliesToPage(c, node, capture) && !(c.check.about === 'console' && !(capture.console_errors || []).length))
           const signals = settings.signals !== false && !!capture.state
           if ((!crit.length && !signals) || capture.criteria_hash !== criteriaHash(node)) return
           asked++
-          await evaluateWithJev({ request: buildRequest({ node, capture, criteria: crit, model: settings.model, signals }), settings, cacheDir, hosted })
+          await text.askAll([{ request: buildRequest({ node, capture, criteria: crit, model: text.model, signals }), asks: crit.length > 0 }], { escalation: !budgeted })
         })
       }
       pump()

package/src/lib/findings.mjs

@@ -0,0 +1,73 @@
+// `kaloko export findings` (#160): the fix list of a run. Every failing acceptance criterion and health check, per
+// page and locale, with the evidence and the way back to the source (edit_url, content_ref of stack pages), as CSV for
+// a content team or JSON for an agent that feeds a content API. Stack steps are read page by page from their items
+// index; the viewports a check fails in are one row.
+import { healthOfNode, levelOf, isHealthCriterion } from './health-core.mjs'
+import { toCsv } from './evidence-core.mjs'
+
+export const FINDING_COLUMNS = ['step', 'page', 'locale', 'viewports', 'url', 'kind', 'pack', 'check', 'level', 'text', 'evidence', 'edit_url', 'content_ref', 'recheck']
+
+const recheckOf = (step, locale, page) => `kaloko walk --steps ${step}${page ? ` --items ${page}` : ''}${locale ? ` --locales ${locale}` : ''}`
+
+/** Rows of failing checks, grouped by step, page, locale and check (viewports joined). */
+function collect (rows, base, results) {
+  const at = new Map()
+  for (const r of results) {
+    const k = `${base.step}|${base.page || ''}|${r.locale}|${r.check}`
+    const row = at.get(k)
+    if (row) { if (!row.viewports.includes(r.viewport)) row.viewports.push(r.viewport); row.evidence ||= r.evidence || null; continue }
+    const next = { ...base, locale: r.locale, viewports: [r.viewport], url: r.url || base.url || null, kind: r.kind, pack: r.pack || null, check: r.check, level: r.level, text: r.text, evidence: r.evidence || null, recheck: recheckOf(base.step, r.locale, base.page) }
+    at.set(k, next); rows.push(next)
+  }
+}
+
+/**
+ * The fix list of a manifest: [{ step, page, locale, viewports, url, kind (acceptance|health), pack, check, level
+ * (error|warning for health, fail for acceptance), text, evidence, edit_url, content_ref, recheck }]. `stackIndex(node)`
+ * returns a stack step's items index (qawalk.stack.v1). `locale` and `step` filter.
+ */
+export async function runFindings (manifest, { stackIndex, locale = null, step = null } = {}) {
+  const rows = []
+  for (const n of manifest?.nodes || []) {
+    if (n.kind === 'decision' || (step && n.id !== step)) continue
+    if (n.kind === 'stack' && n.stack?.items) {
+      const idx = await stackIndex(n)
+      if (!idx) continue
+      const crit = new Map((idx.criteria || []).filter((c) => !c.not_applicable).map((c) => [c.id, c]))
+      for (const it of idx.items || []) {
+        const results = []
+        for (const [key, res] of Object.entries(it.results || {})) {
+          const [l, vp] = key.split('|')
+          if (locale && l !== locale) continue
+          for (const [id, [verdict, evidence]] of Object.entries(res || {})) {
+            const c = crit.get(id)
+            if (!c || verdict !== 'fail') continue
+            results.push({ locale: l, viewport: vp, url: it.captures?.[l]?.[vp]?.url || it.url, check: id, kind: c.health ? 'health' : 'acceptance', pack: c.pack, level: c.health ? (c.level || levelOf(id)) : 'fail', text: c.text, evidence })
+          }
+        }
+        collect(rows, { step: n.id, page: it.id, url: it.url, edit_url: it.edit_url || null, content_ref: it.content_ref || null }, results)
+      }
+      continue
+    }
+    const checks = [...(n.criteria || []).filter((c) => !isHealthCriterion(c)).map((c) => ({ ...c, kind: 'acceptance' })), ...healthOfNode(n).list.map((c) => ({ ...c, kind: 'health' }))]
+    const results = []
+    for (const c of checks) {
+      if (c.not_applicable) continue
+      for (const [l, vps] of Object.entries(c.by_locale || {})) {
+        if (locale && l !== locale) continue
+        for (const [vp, r] of Object.entries(vps || {})) {
+          if (r?.verdict !== 'fail') continue
+          results.push({ locale: l, viewport: vp, url: n.captures?.[l]?.[vp]?.url || null, check: c.id, kind: c.kind, pack: c.pack, level: c.kind === 'health' ? levelOf(c.id) : 'fail', text: c.text, evidence: r.evidence ? String(r.evidence).slice(0, 500) : (r.reason ? String(r.reason).slice(0, 500) : null) })
+        }
+      }
+    }
+    collect(rows, { step: n.id, page: null, url: null, edit_url: null, content_ref: null }, results)
+  }
+  return rows
+}
+
+/** The fix list as CSV (one row per finding, viewports space-separated) or JSON ({ run, scenario, findings }). */
+export function findingsText (rows, format, { run = null, scenario = null } = {}) {
+  if (format === 'json') return JSON.stringify({ schema: 'qawalk.findings.v1', run, scenario, count: rows.length, findings: rows }, null, 2)
+  return toCsv(rows.map((r) => ({ ...r, viewports: r.viewports.join(' ') })), FINDING_COLUMNS)
+}

package/src/lib/health-core.mjs

@@ -16,9 +16,14 @@
 // never rewrite what is stored; an own criterion AC302 without `from` still replaces the pack check HC302, as an
 // adopted check always did.
 //
+// Site checks (#152) run once per site, not per step: the manifest keeps them as `site`, and their findings name the
+// step site:<host>. A cross-page check of a stack (#151, `cross_page`) is a finding of its stack step.
+//
 // Pure: no Node APIs. Shared by the CLI, the service (worker/src/app-health.js) and the tests; the canvas keeps a copy
 // of the few rules it needs (viewer/index.html, hl*).
 
+import { packHelp } from './pack-help.mjs'
+
 /** Packs that count for acceptance unless a scenario says otherwise: the design reference check (#42). */
 export const ACCEPTANCE_PACKS = ['design']
 
@@ -56,7 +61,8 @@ export const CHECK_LEVEL = {
   HC301: 'error', HC302: 'error', HC303: 'error', HC305: 'error', HC310: 'error', HC312: 'error',
   HC501: 'error',
   HC601: 'error', HC604: 'error', HC606: 'error',
-  HC701: 'error', HC704: 'error'
+  HC701: 'error', HC704: 'error',
+  HC1102: 'error'
 }
 export const levelOf = (id) => CHECK_LEVEL[toHealthId(id)] || 'warning'
 
@@ -144,13 +150,30 @@ export function runHealth (manifest) {
       })
     }
   }
+  // site checks (#152): once per site and run, stored as manifest.site; a finding's step is site:<host>
+  const site = []
+  for (const o of manifest?.site?.origins || []) {
+    const host = (() => { try { return new URL(o.origin).host } catch { return String(o.origin) } })()
+    const row = { origin: o.origin, host, checks: 0, error: 0, warning: 0 }
+    for (const c of o.checks || []) {
+      if (c.na || c.verdict === 'not_evaluated') continue
+      checks++; row.checks++
+      const p = (packs.site ||= { checks: 0, error: 0, warning: 0, info: 0 })
+      p.checks++
+      if (c.verdict !== 'fail') continue
+      const severity = c.level || levelOf(c.id)
+      counts[severity]++; p[severity]++; row[severity]++
+      findings.push({ id: findingId(`site:${host}`, c.id), step: `site:${host}`, title: host, check: c.id, pack: 'site', text: c.text, severity, verdict: c.verdict, evidence: c.evidence ? String(c.evidence).slice(0, 300) : null, cells: [], site: true, ...(packHelp(c.id) ? { help: { en: packHelp(c.id, 'en'), cs: packHelp(c.id, 'cs') } } : {}) })
+    }
+    site.push(row)
+  }
   findings.sort((a, b) => severityRank(b.severity) - severityRank(a.severity) || a.step.localeCompare(b.step) || a.check.localeCompare(b.check))
-  return { checks, legacy, counts, packs, findings }
+  return { checks, legacy, counts, packs, findings, ...(site.length ? { site } : {}) }
 }
 
-/** The counts a run summary keeps (summary.health), from the manifest nodes. Null when nothing was checked. */
-export function healthSummary (nodes) {
-  const h = runHealth({ nodes })
+/** The counts a run summary keeps (summary.health), from the manifest nodes and site checks. Null when nothing was checked. */
+export function healthSummary (nodes, site = null) {
+  const h = runHealth({ nodes, site })
   if (!h.checks) return null
   return { checks: h.checks, ...h.counts, packs: h.packs }
 }

package/src/lib/health-index.mjs

@@ -0,0 +1,187 @@
+// The health index of a run (#155, #154): every health check result per step, locale, page and check, folded into
+// groups for dashboards (GET …/runs/<id>/health/aggregate) and the list of results that fail or are unsure, keyed
+// step|locale|check|page so a later run of the same flow can say what is new, fixed or still open (…/health/diff).
+// A page is a stack item (its id); a plain step is its own page (empty page id). Viewports fold into their worst
+// verdict, so one result is one page in one locale for one check.
+//
+// Pure: no Node APIs. The service builds the index once per shared run and keeps it with the run's files
+// (worker/src/health-index.js); the shapes are documented in help/reference/health-api.md.
+import { healthOfNode, levelOf, findingSeverity, severityRank } from './health-core.mjs'
+import { worst } from './manifest-core.mjs'
+
+export const HEALTH_INDEX_SCHEMA = 'kaloko.health-index.v1'
+export const HEALTH_AGGREGATE_SCHEMA = 'kaloko.health-aggregate.v1'
+export const HEALTH_DIFF_SCHEMA = 'kaloko.health-diff.v1'
+/** What a dashboard may group by; step is the template of a stack step. */
+export const AGGREGATE_DIMS = ['locale', 'step', 'check', 'pack']
+export const DEFAULT_BY = ['locale', 'step', 'check']
+export const VERDICTS = ['pass', 'fail', 'needs_review', 'not_evaluated']
+/** Affected pages kept per locale, step and check; an answer shows at most this many per group. */
+export const TOP_KEEP = 20
+/** Open results a run keeps (worst first); a site with more is cut and says so (`truncated`). */
+export const OPEN_MAX = 50000
+
+const blank = () => ({ pass: 0, fail: 0, needs_review: 0, not_evaluated: 0 })
+const cut = (s, n = 200) => s == null ? null : String(s).slice(0, n)
+
+/** The key of one result: stable across runs while the step, locale, check and page keep their ids. */
+export const healthKey = (step, locale, check, page) => [step, locale, check, page].join('|')
+/** step|locale|check|page → { step, locale, check, page }; the page may contain "|" (it is last). */
+export function parseHealthKey (key) {
+  const [step, locale, check, ...rest] = String(key).split('|')
+  return { step, locale, check, page: rest.join('|') }
+}
+
+const firstUrl = (vps) => { for (const c of Object.values(vps || {})) if (c?.url) return c.url; return null }
+
+/**
+ * The index of a run manifest. `stacks` maps a stack step's id to its items index (steps/<node>/items.json); a stack
+ * step without one counts as one page. Returns { schema, checks: { HC…: { text, pack, level } }, totals, groups:
+ * [{ locale, step, check, counts, top: [{ page, path, url, verdict, evidence }] }], open: [[key, severity, url]],
+ * open_total, truncated, passed }. passed says which checks passed on which page: { "step|locale": { sigs: ["HC103,HC112",
+ * …], pages: { <page>: <index into sigs> } } }; only a result that passed counts as fixed later (passedIn).
+ */
+export function buildHealthIndex (manifest, stacks = {}) {
+  const checks = Object.create(null); const groups = new Map(); const open = []; const pages = new Set(); const ok = new Map()
+  const totals = { results: 0, ...blank() }
+  const add = (step, locale, page, path, url, check, verdict, evidence) => {
+    if (!VERDICTS.includes(verdict)) return
+    pages.add(`${step}|${page}`)
+    const gk = `${locale}\u0000${step}\u0000${check}`
+    let g = groups.get(gk)
+    if (!g) { g = { locale, step, check, counts: blank(), fails: [], unsure: [] }; groups.set(gk, g) }
+    g.counts[verdict]++; totals[verdict]++; totals.results++
+    if (verdict === 'pass') { const k = `${step}|${locale}\u0000${page}`; if (!ok.has(k)) ok.set(k, []); ok.get(k).push(check) }
+    const severity = findingSeverity(verdict, check)
+    if (!severity) return
+    const list = verdict === 'fail' ? g.fails : g.unsure
+    if (list.length < TOP_KEEP) list.push({ page, ...(path ? { path } : {}), url: url || null, verdict, ...(evidence ? { evidence: cut(evidence) } : {}) })
+    open.push([healthKey(step, locale, check, page), severity, url || null])
+  }
+  for (const n of manifest?.nodes || []) {
+    if (n.kind === 'decision') continue
+    const live = healthOfNode(n).list.filter((c) => !c.not_applicable)
+    if (!live.length) continue
+    for (const c of live) checks[c.id] ||= { text: c.text || null, pack: c.pack || 'other', level: c.level || levelOf(c.id) }
+    const idx = n.kind === 'stack' && Object.hasOwn(stacks, n.id) ? stacks[n.id] : null
+    if (idx?.items) {
+      // the items index keeps a check under the id it was walked with (AC302 before the HC ids)
+      const ids = new Map(live.map((c) => [c.stored_id || c.id, c.id]))
+      for (const it of idx.items) {
+        const per = new Map()
+        for (const [cell, r] of Object.entries(it.results || {})) {
+          const locale = cell.split('|')[0]
+          for (const [stored, id] of ids) {
+            const x = r?.[stored]
+            if (!x) continue
+            const k = `${locale}\u0000${id}`
+            const was = per.get(k)
+            if (!was || worst([was.v, x[0]]) !== was.v) per.set(k, { locale, id, v: x[0], e: x[1] })
+          }
+        }
+        for (const { locale, id, v, e } of per.values()) add(n.id, locale, String(it.id), it.path || null, it.url || firstUrl(it.captures?.[locale]), id, v, e)
+      }
+      continue
+    }
+    for (const c of live) {
+      const locales = Object.entries(c.by_locale || {})
+      if (!locales.length) { add(n.id, '', '', n.path || null, null, c.id, c.verdict, c.evidence || c.reason); continue }
+      for (const [locale, vps] of locales) {
+        const rs = Object.values(vps || {}).filter(Boolean)
+        const v = worst(rs.map((r) => r.verdict).filter(Boolean))
+        const bad = rs.find((r) => r.verdict === v)
+        add(n.id, locale, '', n.path || null, firstUrl(n.captures?.[locale]), c.id, v, bad?.evidence || bad?.reason)
+      }
+    }
+  }
+  open.sort((a, b) => severityRank(b[1]) - severityRank(a[1]) || (a[0] < b[0] ? -1 : 1))
+  // pages that passed the same checks share one signature: a stack of 400 pages keeps a handful of strings
+  // null-prototype objects: a step or page id such as "constructor" is data, never a property of Object
+  const passed = Object.create(null)
+  for (const [k, list] of ok) {
+    const [where, page] = k.split('\u0000')
+    const w = (passed[where] ||= { sigs: [], pages: Object.create(null) })
+    const sig = [...new Set(list)].sort().join(',')
+    let i = w.sigs.indexOf(sig)
+    if (i < 0) { i = w.sigs.length; w.sigs.push(sig) }
+    w.pages[page] = i
+  }
+  return {
+    schema: HEALTH_INDEX_SCHEMA, checks, totals: { ...totals, pages: pages.size },
+    groups: [...groups.values()].map(({ fails, unsure, ...g }) => ({ ...g, top: [...fails, ...unsure].slice(0, TOP_KEEP) })),
+    open: open.slice(0, OPEN_MAX), open_total: open.length, truncated: open.length > OPEN_MAX, passed
+  }
+}
+
+/** `?by=locale,step` → the dimensions in canonical order; unknown names throw. Empty → the default. */
+export function parseBy (value) {
+  const list = String(value ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+  const bad = list.find((d) => !AGGREGATE_DIMS.includes(d))
+  if (bad) throw new Error(`Unknown dimension "${bad.slice(0, 30)}": group by ${AGGREGATE_DIMS.join(', ')}.`)
+  return list.length ? AGGREGATE_DIMS.filter((d) => list.includes(d)) : DEFAULT_BY
+}
+
+/**
+ * Counts per group of an index: `by` dimensions, `filter` { locale, step, check, pack } (exact values), up to `top`
+ * affected pages per group (fails first) and `limit` groups, the most failing first. Group ids are the dimension
+ * values ("locale=de;step=eshop;check=HC112", "all" without dimensions), stable across runs.
+ */
+export function aggregateHealth (index, { by = DEFAULT_BY, filter = {}, top = 5, limit = 500 } = {}) {
+  const out = new Map(); const totals = { results: 0, ...blank() }
+  const want = Object.entries(filter).filter(([k, v]) => AGGREGATE_DIMS.includes(k) && v != null && v !== '')
+  const keep = Math.min(TOP_KEEP, Math.max(0, Number(top) || 0))
+  for (const g of index?.groups || []) {
+    const vals = { locale: g.locale, step: g.step, check: g.check, pack: index.checks?.[g.check]?.pack || 'other' }
+    if (want.some(([k, v]) => vals[k] !== String(v))) continue
+    const id = by.map((d) => `${d}=${vals[d]}`).join(';') || 'all'
+    let o = out.get(id)
+    if (!o) { o = { id, ...Object.fromEntries(by.map((d) => [d, vals[d]])), results: 0, counts: blank(), affected: 0, top: [] }; out.set(id, o) }
+    for (const v of VERDICTS) { o.counts[v] += g.counts[v] || 0; totals[v] += g.counts[v] || 0 }
+    if (keep) for (const t of g.top) o.top.push({ step: g.step, locale: g.locale, check: g.check, ...t })
+  }
+  const groups = [...out.values()].map((o) => {
+    o.results = VERDICTS.reduce((s, v) => s + o.counts[v], 0)
+    o.affected = o.counts.fail + o.counts.needs_review
+    o.top = o.top.sort((a, b) => (a.verdict === 'fail' ? 0 : 1) - (b.verdict === 'fail' ? 0 : 1)).slice(0, keep)
+    return o
+  }).sort((a, b) => b.counts.fail - a.counts.fail || b.counts.needs_review - a.counts.needs_review || (a.id < b.id ? -1 : 1))
+  totals.results = VERDICTS.reduce((s, v) => s + totals[v], 0)
+  const max = Math.max(1, Number(limit) || 500)
+  return { by, totals, group_count: groups.length, truncated: groups.length > max, groups: groups.slice(0, max) }
+}
+
+/** True when the run checked this result (step|locale|check|page) and it passed. */
+export function passedIn (index, key) {
+  const { step, locale, check, page } = parseHealthKey(key)
+  const where = `${step}|${locale}`
+  // a stored index comes back from JSON with plain objects: own keys only
+  const w = index?.passed && Object.hasOwn(index.passed, where) ? index.passed[where] : null
+  const i = w?.pages && Object.hasOwn(w.pages, page) ? w.pages[page] : null
+  return Number.isInteger(i) && typeof w.sigs?.[i] === 'string' && w.sigs[i].split(',').includes(check)
+}
+
+const entry = ([key, severity, url]) => ({ id: key, ...parseHealthKey(key), severity: severity || null, url: url || null })
+
+/**
+ * What changed between the open results of `before` ([[key, severity, url]]: the previous run's open results, or what
+ * the flow keeps open) and the index `cur`: new (open now, not before), still open (both; `was` names the earlier
+ * severity when it changed), fixed (open before, checked now and passed) and unchecked (open before, not checked now:
+ * a rotating sample, a locale left out, an unsure evaluator, a cut list; it stays open). Without `before` everything
+ * open is new and `previous` is false.
+ */
+export function diffHealth (cur, before) {
+  const now = new Map((cur?.open || []).map((o) => [o[0], o]))
+  if (!before) return { previous: false, new: [...now.values()].map(entry), fixed: [], still_open: [], unchecked: [] }
+  const prev = new Map(before.map((o) => [o[0], o]))
+  const fresh = []; const still = []; const fixed = []; const unchecked = []
+  for (const [k, o] of now) {
+    const b = prev.get(k)
+    if (!b) fresh.push(entry(o))
+    else still.push({ ...entry(o), ...(b[1] && b[1] !== o[1] ? { was: b[1] } : {}) })
+  }
+  for (const [k, o] of prev) if (!now.has(k)) (passedIn(cur, k) ? fixed : unchecked).push(entry(o))
+  return { previous: true, new: fresh, fixed, still_open: still, unchecked }
+}
+
+/** Diff counts: new, fixed, still open, unchecked and how many still open got worse. */
+export const diffCounts = (d) => ({ new: d.new.length, fixed: d.fixed.length, still_open: d.still_open.length, unchecked: d.unchecked.length, worse: d.still_open.filter((e) => e.was && severityRank(e.severity) > severityRank(e.was)).length })