kaloko 0.3.0

package/src/lib/calibrate.mjs

@@ -0,0 +1,97 @@
+// Calibration of the semantic evaluator (#10): compare evaluator scores with human labels, report precision and
+// recall per criterion at the configured thresholds, and suggest thresholds that keep automatic verdicts precise.
+// Labels come from a golden file (local runs) or from reviewers' verdicts on criteria in the service.
+import { readFile } from 'node:fs/promises'
+import { resolve, dirname, join } from 'node:path'
+import yaml from 'js-yaml'
+
+const GRID = Array.from({ length: 19 }, (_, i) => Math.round((0.05 + i * 0.05) * 100) / 100)
+
+/** Verdict of a score under thresholds { pass, fail }: pass, fail or needs_review (the band between). */
+const decide = (score, t) => (score >= t.pass ? 'pass' : score <= t.fail ? 'fail' : 'needs_review')
+
+function metrics (cases, t) {
+  const n = cases.length
+  const by = (pred, exp) => cases.filter((c) => decide(c.score, t) === pred && (exp == null || c.expected === exp)).length
+  const pp = by('pass'); const pf = by('fail'); const tp = by('pass', 'pass'); const tf = by('fail', 'fail')
+  const ep = cases.filter((c) => c.expected === 'pass').length; const ef = n - ep
+  const ratio = (a, b) => (b ? Math.round((a / b) * 1000) / 1000 : null)
+  return {
+    n, expected_pass: ep, expected_fail: ef,
+    pass_precision: ratio(tp, pp), pass_recall: ratio(tp, ep),
+    fail_precision: ratio(tf, pf), fail_recall: ratio(tf, ef),
+    needs_review: ratio(n - pp - pf, n),
+    wrong: pp - tp + pf - tf
+  }
+}
+
+/**
+ * Suggested thresholds: the lowest pass threshold whose automatic passes are at least `target` precise and the
+ * highest fail threshold whose automatic fails are, with fail < pass. Null when the labels do not support one.
+ */
+export function suggest (cases, { target = 0.95 } = {}) {
+  const precise = (sel, exp) => { const s = cases.filter(sel); return s.length > 0 && s.filter((c) => c.expected === exp).length / s.length >= target }
+  // widest precise band, then tightened to the scores actually observed so it does not reach into gaps without data
+  const pmin = GRID.find((t) => precise((c) => c.score >= t, 'pass'))
+  const pass = pmin == null ? null : [...GRID].reverse().find((t) => t <= Math.min(...cases.filter((c) => c.score >= pmin).map((c) => c.score)))
+  const fmax = GRID.filter((t) => precise((c) => c.score <= t, 'fail') && (pass == null || t < pass)).pop()
+  const fail = fmax == null ? null : GRID.find((t) => t >= Math.max(...cases.filter((c) => c.score <= fmax).map((c) => c.score)))
+  return { pass: pass ?? null, fail: fail != null && (pass == null || fail < pass) ? fail : null }
+}
+
+/** cases: [{ criterion, score, expected: 'pass'|'fail', ref? }] → per criterion and overall report. */
+export function calibrate (cases, thresholds, { target = 0.95 } = {}) {
+  const usable = cases.filter((c) => typeof c.score === 'number' && (c.expected === 'pass' || c.expected === 'fail'))
+  const groups = new Map()
+  for (const c of usable) { if (!groups.has(c.criterion)) groups.set(c.criterion, []); groups.get(c.criterion).push(c) }
+  const criteria = [...groups.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([criterion, list]) => ({
+    criterion, ...metrics(list, thresholds), suggested: suggest(list, { target }),
+    misses: list.filter((c) => { const d = decide(c.score, thresholds); return d !== 'needs_review' && d !== c.expected }).map((c) => ({ ref: c.ref || null, score: c.score, expected: c.expected }))
+  }))
+  return { thresholds, target, skipped: cases.length - usable.length, overall: { ...metrics(usable, thresholds), suggested: suggest(usable, { target }) }, criteria }
+}
+
+const pct = (v) => (v == null ? '  —  ' : `${(v * 100).toFixed(0).padStart(3)} %`)
+export function formatCalibration (r) {
+  const row = (name, m) => `${name.padEnd(28).slice(0, 28)} ${String(m.n).padStart(4)}  ${pct(m.pass_precision)} ${pct(m.pass_recall)}  ${pct(m.fail_precision)} ${pct(m.fail_recall)}  ${pct(m.needs_review)}  ${m.suggested.pass ?? '—'} / ${m.suggested.fail ?? '—'}`
+  const lines = [
+    `Thresholds pass ≥ ${r.thresholds.pass}, fail ≤ ${r.thresholds.fail}; suggestions keep automatic verdicts ≥ ${Math.round(r.target * 100)} % precise.`,
+    `${'criterion'.padEnd(28)}    n  pass P  pass R  fail P  fail R  review   suggest pass / fail`,
+    ...r.criteria.map((c) => row(c.criterion, c)),
+    row('all', r.overall)
+  ]
+  const misses = r.criteria.flatMap((c) => c.misses.map((m) => `  ${c.criterion}: ${m.ref || ''} score ${m.score}, labelled ${m.expected}`))
+  if (misses.length) lines.push('', 'Automatic verdicts that contradict the labels:', ...misses.slice(0, 30))
+  if (r.skipped) lines.push('', `${r.skipped} labelled case(s) had no evaluator score and were skipped.`)
+  return lines.join('\n')
+}
+
+/**
+ * Golden file: `cases: [{ run: <run dir>, step, locale, viewport, criterion, expected }]`, paths relative to the
+ * file. Scores are read from the stored evaluations (<run>/steps/<step>/<locale>/<viewport>.eval.json).
+ */
+export async function casesFromGolden (file) {
+  const doc = yaml.load(await readFile(file, 'utf8')) || {}
+  const base = dirname(resolve(file))
+  const out = []
+  for (const k of doc.cases || []) {
+    const ev = JSON.parse(await readFile(join(resolve(base, k.run), 'steps', k.step, k.locale, `${k.viewport}.eval.json`), 'utf8').catch(() => 'null'))
+    const c = ev?.criteria?.find((x) => x.id === k.criterion)
+    out.push({ criterion: `${k.step}/${k.criterion}`, score: typeof c?.score === 'number' ? c.score : null, expected: k.expected, ref: `${k.run} ${k.step} ${k.locale}/${k.viewport}` })
+  }
+  return out
+}
+
+/** Labels from the service: reviewers' pass/fail verdicts on criteria, matched with the evaluator scores in the manifests. */
+export function casesFromRun (manifest, verdicts) {
+  const out = []
+  for (const v of verdicts || []) {
+    const node = manifest.nodes?.find((n) => n.id === v.node_id)
+    const crit = node?.criteria?.find((c) => c.id === v.criterion_id)
+    if (!crit || crit.type !== 'semantic') continue
+    for (const [vp, r] of Object.entries(crit.by_locale?.[v.locale] || {})) {
+      if (typeof r.score === 'number') out.push({ criterion: `${node.id}/${crit.id}`, score: r.score, expected: v.verdict, ref: `${manifest.id} ${node.id} ${v.locale}/${vp}` })
+    }
+  }
+  return out
+}

package/src/lib/capture.mjs

@@ -0,0 +1,233 @@
+import { writeFile, mkdir, rm } from 'node:fs/promises'
+import { dirname, relative, join } from 'node:path'
+import { buildHtmlView } from './htmlview.mjs'
+import { probe } from './probe.mjs'
+import { writeJson, readJsonl } from './util.mjs'
+import { addCapture, runPaths } from './run.mjs'
+
+/**
+ * Encodes a PNG screenshot inside the page (no native dependency): the image as WebP (or PNG, lossless) and a
+ * WebP thumbnail for the canvas grid; `marks` are numbered circles drawn on the image (documentation captures).
+ */
+export async function encodeImage (page, png, { format = 'webp', quality = 0.82, thumbWidth = 320, thumbQuality = 0.7, marks = [], scale = 1, masks = [] } = {}) {
+  const result = await page.evaluate(async ({ dataUrl, format, quality, thumbWidth, thumbQuality, marks, scale, masks }) => {
+    const img = new Image()
+    await new Promise((res, rej) => { img.onload = res; img.onerror = rej; img.src = dataUrl })
+    const full = document.createElement('canvas'); full.width = img.width; full.height = img.height
+    const g = full.getContext('2d'); g.drawImage(img, 0, 0)
+    g.fillStyle = '#1f1f1d'; for (const r of masks) g.fillRect(Math.floor(r.x * scale), Math.floor(r.y * scale), Math.ceil(r.w * scale), Math.ceil(r.h * scale))
+    for (const m of marks) {
+      const r = 13 * scale; const x = Math.max(r + 2, m.x * scale); const y = Math.max(r + 2, m.y * scale)
+      g.beginPath(); g.arc(x, y, r, 0, Math.PI * 2); g.fillStyle = '#ffe452'; g.fill(); g.lineWidth = 2 * scale; g.strokeStyle = '#151514'; g.stroke()
+      g.fillStyle = '#151514'; g.font = `700 ${14 * scale}px system-ui, sans-serif`; g.textAlign = 'center'; g.textBaseline = 'middle'; g.fillText(String(m.n), x, y + scale)
+    }
+    const blob = (c, type, q) => new Promise((res, rej) => c.toBlob((b) => (b ? b.arrayBuffer().then((buf) => res(Array.from(new Uint8Array(buf)))) : rej(new Error('encode failed'))), type, q))
+    const k = Math.min(1, thumbWidth / img.width)
+    const t = document.createElement('canvas'); t.width = Math.round(img.width * k); t.height = Math.round(img.height * k)
+    t.getContext('2d').drawImage(full, 0, 0, t.width, t.height)
+    return { image: await blob(full, format === 'png' ? 'image/png' : 'image/webp', quality), thumb: await blob(t, 'image/webp', thumbQuality) }
+  }, { dataUrl: `data:image/png;base64,${png.toString('base64')}`, format, quality, thumbWidth, thumbQuality, marks, scale, masks })
+  return { image: Buffer.from(result.image), thumb: Buffer.from(result.thumb) }
+}
+
+/** WebP encoding for the canvas (kept for callers of the old name). */
+export const toWebp = (page, png, opts = {}) => encodeImage(page, png, { ...opts, format: 'webp' })
+
+/**
+ * Capture options: scenario-wide `capture:` merged with the node's. Documentation runs use
+ * { format: png, scale: 2, mask: [selectors], annotate: true }; daily acceptance keeps the WebP default.
+ */
+export function captureOptions (scenario, node) {
+  const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
+  const q = Number(o.quality)
+  return {
+    format: o.format === 'png' ? 'png' : 'webp',
+    scale: Number(o.scale) === 2 ? 2 : 1,
+    quality: Number.isFinite(q) && q > 0 ? Math.min(1, Math.max(0.3, q)) : 0.82,
+    mask: Array.isArray(o.mask) ? o.mask.map(String).filter(Boolean) : [],
+    annotate: o.annotate === true,
+    // stack pages default to the first screen: hundreds of full-page shots would swamp storage
+    fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack'
+  }
+}
+
+import { criteriaHash } from './evaluate.mjs'
+import { consoleEntriesFor } from './console.mjs'
+import { redact, redactDeep } from './secrets.mjs'
+import { readFields } from './stack.mjs'
+
+/** Probe arguments derived from the node's criteria. */
+export function probeArgs (node, maxChars, design = false) {
+  const criteria = node.criteria || []
+  return {
+    maxChars,
+    // design summary (structure + used styles) for design steps and for checks against the design reference
+    design: design || criteria.some((c) => ['uses_tokens', 'design_pixels', 'design_structure'].includes(c.check.assert)),
+    checks: criteria.filter((c) => c.check.type === 'deterministic').map((c) => ({ id: c.id, check: c.check })),
+    highlights: criteria.map((c) => ({ id: c.id, selector: c.highlight || c.check.selector || c.check.scope })).filter((h) => h.selector),
+    scopes: criteria.filter((c) => c.check.type === 'semantic' && c.check.scope).map((c) => ({ id: c.id, selector: c.check.scope }))
+  }
+}
+
+// Open overlays that many UI libraries close on scroll or resize (MUI Menu/Popover, Angular Material
+// md-menu, listboxes, tooltips). A step that leaves one open wants it on the capture.
+const OVERLAY = '[role=menu], [role=listbox], [role=tooltip], .MuiPopover-root, .MuiPopper-root, .MuiMenu-root, md-menu-content, .md-open-menu-container.md-active, .md-panel-outer-wrapper.md-panel-is-showing'
+
+export async function settle (page) {
+  try { await page.waitForLoadState('networkidle', { timeout: 5000 }) } catch {}
+  try { await page.evaluate(() => document.fonts?.ready) } catch {}
+  // walk the page once so lazy images and sticky headers settle, unless an overlay is open:
+  // scrolling would close it and the capture would miss what the step opened
+  try {
+    await page.evaluate(async (overlay) => {
+      const open = [...document.querySelectorAll(overlay)].some((el) => { const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0 })
+      if (open) return
+      const step = innerHeight * 0.8
+      for (let y = 0; y < document.documentElement.scrollHeight; y += step) { scrollTo(0, y); await new Promise((r) => setTimeout(r, 60)) }
+      scrollTo(0, 0)
+    }, OVERLAY)
+  } catch {}
+  await page.waitForTimeout(250)
+}
+
+/**
+ * Captures the current state of `page` for one step and locale in every
+ * requested viewport: full-page PNG, rendered HTML, probe results.
+ */
+export async function captureStep ({ page, node, locale, viewports, run, consoleFile, maxChars = 90000, since, extra = {}, format = null, options = null, designSummary = false, htmlView = null, item = null, fields = null }) {
+  const opts = { ...(options || captureOptions(null, node)), ...(format ? { format } : {}) }
+  // a page of a stack step lives under steps/<node>/items/<item>/…; its capture record names node and item
+  const stepId = item ? `${node.id}/items/${item.id}` : node.id
+  const paths = runPaths(run.dir)
+  const original = page.viewportSize()
+  const results = []
+  for (const [viewport, [width, height]] of Object.entries(viewports)) {
+    const now = page.viewportSize()
+    if (!now || now.width !== width || now.height !== height) await page.setViewportSize({ width, height })
+    await settle(page)
+    const html = paths.step(stepId, locale, viewport, 'html')
+    await rm(paths.step(stepId, locale, viewport, 'eval.json'), { force: true })
+    await mkdir(dirname(html), { recursive: true })
+    // PII selectors are covered by a solid box; at 2× Chrome renders the full page at double scale (CDP), and
+    // the boxes are drawn while encoding because Playwright's own mask only works at the page's pixel ratio
+    const maskRects = opts.mask.length ? await page.evaluate((sels) => sels.flatMap((sel) => { try { return [...document.querySelectorAll(sel)] } catch { return [] } }).map((el) => { const r = el.getBoundingClientRect(); return { x: r.left + scrollX, y: r.top + scrollY, w: r.width, h: r.height } }).filter((r) => r.w > 0 && r.h > 0), opts.mask) : []
+    let shot
+    if (opts.scale === 2) {
+      const cdp = await page.context().newCDPSession(page)
+      try {
+        const doc = await page.evaluate(() => ({ w: Math.ceil(document.documentElement.scrollWidth), h: Math.ceil(document.documentElement.scrollHeight) }))
+        const r = await cdp.send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: opts.fullPage, clip: { x: 0, y: 0, width: opts.fullPage ? doc.w : width, height: opts.fullPage ? doc.h : height, scale: 2 } })
+        shot = Buffer.from(r.data, 'base64')
+      } finally { await cdp.detach().catch(() => {}) }
+    } else shot = await page.screenshot({ fullPage: opts.fullPage, ...(opts.mask.length ? { mask: opts.mask.map((sel) => page.locator(sel)), maskColor: '#1f1f1d' } : {}) })
+    // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
+    await writeFile(html, redact(await page.content()))
+    let result
+    for (let attempt = 0; ; attempt++) {
+      try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
+        // the page navigated under us (redirect, Turbo); wait once and probe the new document
+        if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
+        throw error
+      }
+    }
+    // numbered marks on the highlighted elements, in criteria order (first box of each)
+    const annotations = opts.annotate ? Object.entries(result.boxes || {}).filter(([, list]) => list?.length).map(([criterion, list], i) => ({ n: i + 1, criterion, x: list[0].x, y: list[0].y })) : []
+    let image = paths.step(stepId, locale, viewport, opts.format)
+    const thumb = paths.step(stepId, locale, viewport, 'thumb.webp')
+    let thumbWritten = true
+    try {
+      const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: opts.scale, masks: opts.scale === 2 ? maskRects : [] })
+      await writeFile(image, enc.image)
+      await writeFile(thumb, enc.thumb)
+    } catch (error) {
+      // never store an unmasked image when masks were asked for
+      if (opts.scale === 2 && maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
+      image = paths.step(stepId, locale, viewport, 'png'); thumbWritten = false
+      await writeFile(image, shot)
+    }
+    const url = page.url()
+    const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
+    const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
+    // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
+    // an error page's own status ("Failed to load resource: … 404" for the document itself) is not a console error
+    const ownStatus = (e) => result.http_status >= 400 && /^Failed to load resource: the server responded with a status of \d+/.test(e.text) && e.src && sameDocument(e.src, url)
+    const errors = consoleEntries.filter((e) => e.type !== 'blocked' && sameDocument(e.url, url) && !e.text.includes('ERR_BLOCKED_BY_CLIENT') && !ownStatus(e)).map((e) => e.text)
+    const blocked = consoleEntries.filter((e) => e.type === 'blocked').map((e) => `${e.text}`)
+    for (const c of node.criteria || []) {
+      if (c.check.type === 'deterministic' && c.check.assert === 'no_console_errors') {
+        result.checks[c.id] = collected === null
+          ? { ok: null, evidence: 'Console was not observed.' }
+          : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
+      }
+    }
+    // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
+    let view = null
+    if (htmlView) {
+      try {
+        // the same PII masks as the screenshot (#27), applied to the text of the copy
+        const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])] })
+        await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
+        view = { revision: rev.id, entry: rev.entry }
+      } catch (error) { view = { error: String(error.message).slice(0, 200) } }
+    }
+    const record = {
+      schema: 'qawalk.capture.v1',
+      ...(view ? { view } : {}),
+      node: node.id, locale, viewport, url,
+      captured_at: new Date().toISOString(),
+      criteria_hash: criteriaHash(node),
+      console_errors: errors, blocked_requests: blocked,
+      ...result,
+      ...extra,
+      http_status: extra.http_status ?? result.http_status ?? null,
+      capture: { format: String(image).split('.').pop(), scale: opts.scale, masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+      ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}) } } : {}),
+      ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
+      files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }
+    }
+    const file = paths.step(stepId, locale, viewport, 'capture.json')
+    await writeJson(file, redactDeep(record))
+    await addCapture(run.dir, { type: 'capture', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, file: relative(run.dir, file), at: record.captured_at })
+    results.push(record)
+  }
+  const last = page.viewportSize()
+  if (original && (!last || last.width !== original.width || last.height !== original.height)) await page.setViewportSize(original)
+  return results
+}
+
+export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0 }) {
+  const paths = runPaths(run.dir)
+  const el = page.locator(selector).first()
+  const name = `${node.id}-${locale}-${Date.now().toString(36)}.png`
+  const file = paths.attachment(name)
+  await mkdir(dirname(file), { recursive: true })
+  if (padding > 0) {
+    // element plus a margin of surrounding page, clipped from a full-page shot (documentation crops)
+    const box = await el.boundingBox()
+    if (!box) throw new Error(`crop: ${selector} is not visible`)
+    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
+    const x = Math.max(0, box.x + doc.x - padding); const y = Math.max(0, box.y + doc.y - padding)
+    await page.screenshot({ path: file, fullPage: true, clip: { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) } })
+  } else await el.screenshot({ path: file })
+  const entry = { type: 'attachment', kind: 'crop', node: node.id, locale, title: title || selector, selector, file: relative(run.dir, file), viewport: page.viewportSize(), at: new Date().toISOString() }
+  await addCapture(run.dir, entry)
+  return entry
+}
+
+function sameDocument (a, b) {
+  try {
+    const x = new URL(a); const y = new URL(b)
+    return x.origin === y.origin && x.pathname === y.pathname
+  } catch { return a === b }
+}
+
+/**
+ * Accepts native dialogs (confirm, alert, prompt) in the page from now on, also after navigations. With several
+ * Playwright clients on the shared browser over CDP, a client without a dialog handler can dismiss a native dialog
+ * before the step's handler accepts it (#53); replacing the functions in the page avoids the race entirely.
+ */
+export async function acceptDialogs (page) {
+  const stub = () => { window.confirm = () => true; window.alert = () => {}; window.prompt = (_m, d) => (d == null ? '' : String(d)) }
+  await page.addInitScript(stub)
+  await page.evaluate(stub).catch(() => {})
+}

package/src/lib/compare.mjs

@@ -0,0 +1,88 @@
+// Compares two run manifests of the same scenario: step statuses, criterion
+// verdicts and evidence per locale and viewport, and which captures exist on
+// both sides (the canvas computes pixel diffs from those in the browser).
+
+const RANK = { fail: 4, needs_review: 3, not_evaluated: 2, pass: 1, not_reached: 0 }
+const worst = (list) => list.reduce((a, b) => (RANK[b] > RANK[a] ? b : a), 'not_reached')
+const text = (v, locale) => v == null ? '' : typeof v === 'string' ? v : (v[locale] ?? v.default ?? Object.values(v)[0] ?? '')
+
+const brief = (r) => ({ id: r.id, started_at: r.started_at, environment: r.environment?.name || null, git: r.git ? { branch: r.git.branch, commit: r.git.commit } : null, summary: r.summary || {} })
+
+/** Result of one criterion in one locale and viewport, or null when the step was not reached. */
+const cell = (c, locale, vp) => {
+  const r = c.by_locale?.[locale]?.[vp]
+  if (!r || !r.verdict || r.verdict === 'not_reached') return null
+  return { verdict: r.verdict, evidence: r.evidence || r.note || null, score: r.score ?? null }
+}
+
+/**
+ * @returns {{ run, base, same_scenario, nodes: Array, added: string[], removed: string[], changed: number }}
+ * nodes: { id, title, status: {run, base}, changed, criteria: [{ id, text, changed, cells: [{ locale, viewport, run, base }] }], captures: [{ locale, viewport, run, base }] }
+ */
+export function compareRuns (run, base) {
+  const locales = [...new Set([...(run.scope?.locales || []), ...(base.scope?.locales || [])])]
+  const viewports = [...new Set([...Object.keys(run.scope?.viewports || {}), ...Object.keys(base.scope?.viewports || {})])]
+  const baseNodes = new Map((base.nodes || []).map((n) => [n.id, n]))
+  const runNodes = new Map((run.nodes || []).map((n) => [n.id, n]))
+  const nodes = []
+  for (const n of run.nodes || []) {
+    if (n.kind === 'decision') continue
+    const b = baseNodes.get(n.id)
+    if (!b) continue
+    const baseCrit = new Map((b.criteria || []).map((c) => [c.id, c]))
+    const criteria = []
+    for (const c of n.criteria || []) {
+      const bc = baseCrit.get(c.id)
+      const cells = []
+      for (const locale of locales) {
+        for (const vp of viewports) {
+          const a = cell(c, locale, vp), z = bc ? cell(bc, locale, vp) : null
+          if (!a && !z) continue
+          const changed = (a?.verdict || null) !== (z?.verdict || null) || (a?.evidence || '') !== (z?.evidence || '')
+          if (changed) cells.push({ locale, viewport: vp, run: a, base: z })
+        }
+      }
+      // one verdict per criterion: worst over locales and viewports
+      const all = (c2) => c2 ? locales.flatMap((l) => viewports.map((vp) => cell(c2, l, vp)?.verdict).filter(Boolean)) : []
+      const vRun = all(c), vBase = all(bc)
+      criteria.push({ id: c.id, text: c.text, verdict: { run: vRun.length ? worst(vRun) : null, base: vBase.length ? worst(vBase) : null }, cells, changed: cells.length > 0 })
+    }
+    const captures = []
+    for (const locale of locales) for (const vp of viewports) {
+      const a = n.captures?.[locale]?.[vp], z = b.captures?.[locale]?.[vp]
+      if (a || z) captures.push({ locale, viewport: vp, run: a ? (a.image || a.png) : null, base: z ? (z.image || z.png) : null, url: a?.url || z?.url || null })
+    }
+    const status = { run: n.status || null, base: b.status || null }
+    nodes.push({ id: n.id, title: text(n.title, locales[0]), status, changed: status.run !== status.base || criteria.some((c) => c.changed), criteria, captures })
+  }
+  const added = [...runNodes.keys()].filter((id) => !baseNodes.has(id) && runNodes.get(id).kind !== 'decision')
+  const removed = [...baseNodes.keys()].filter((id) => !runNodes.has(id) && baseNodes.get(id).kind !== 'decision')
+  return { run: brief(run), base: brief(base), same_scenario: run.scenario?.id === base.scenario?.id, nodes, added, removed, changed: nodes.filter((n) => n.changed).length + added.length + removed.length }
+}
+
+/** Plain-text report for the terminal and issue comments. */
+export function formatComparison (d) {
+  const sum = (s) => `${s.pass || 0} pass · ${s.partial || 0} partial · ${s.fail || 0} fail`
+  const who = (r) => `${r.id} (${[r.environment, r.git && `${r.git.branch} @ ${r.git.commit}`].filter(Boolean).join(', ')})`
+  const out = [`Compare ${who(d.run)}\n   with ${who(d.base)}`]
+  if (!d.same_scenario) out.push('  ! different scenarios')
+  out.push(`Steps: ${sum(d.run.summary)}  ←  ${sum(d.base.summary)}`)
+  for (const n of d.nodes.filter((x) => x.changed)) {
+    out.push(`  ${n.id}: ${n.status.base || '—'} → ${n.status.run || '—'}${n.title ? `  (${n.title})` : ''}`)
+    for (const c of n.criteria.filter((x) => x.changed)) {
+      for (const k of c.cells) {
+        const v = (k.base?.verdict || '—') !== (k.run?.verdict || '—') ? `${k.base?.verdict || '—'} → ${k.run?.verdict || '—'}` : k.run?.verdict || '—'
+        const score = k.base?.score != null || k.run?.score != null ? ` (${k.base?.score != null ? k.base.score.toFixed(2) : '—'} → ${k.run?.score != null ? k.run.score.toFixed(2) : '—'})` : ''
+        const cut = (t) => { t = String(t || ''); return t.length > 100 ? t.slice(0, 97) + '…' : t }
+        const ev = (k.base?.evidence || '') !== (k.run?.evidence || '') ? `\n        evidence: ${JSON.stringify(cut(k.base?.evidence))} → ${JSON.stringify(cut(k.run?.evidence))}` : ''
+        out.push(`    ${c.id} [${k.locale}/${k.viewport}]: ${v}${score}${ev}`)
+      }
+    }
+  }
+  if (d.added.length) out.push(`  added steps: ${d.added.join(', ')}`)
+  if (d.removed.length) out.push(`  removed steps: ${d.removed.join(', ')}`)
+  const same = d.nodes.filter((x) => !x.changed).map((x) => x.id)
+  if (same.length) out.push(`  unchanged: ${same.join(', ')}`)
+  if (!d.changed) out.push('No differences in statuses, verdicts or evidence.')
+  return out.join('\n')
+}

package/src/lib/config.mjs

@@ -0,0 +1,117 @@
+import { readFile, access } from 'node:fs/promises'
+import { join, dirname, resolve } from 'node:path'
+import { homedir } from 'node:os'
+import yaml from 'js-yaml'
+import { schemaErrors } from './schema.mjs'
+import { UserError, localized, globToRegExp } from './util.mjs'
+
+export const CONFIG_FILE = 'kaloko.config.yml'
+/** The file name before the rename to Kaloko; still found, so existing projects keep working. */
+export const LEGACY_CONFIG_FILE = 'qawalk.config.yml'
+
+/** Environment variables were QAWALK_*; the old names still work and fill the KALOKO_* ones that are not set. */
+export function aliasLegacyEnv (vars = process.env) {
+  for (const [k, v] of Object.entries(vars)) if (k.startsWith('QAWALK_') && vars[`KALOKO_${k.slice(7)}`] == null) vars[`KALOKO_${k.slice(7)}`] = v
+}
+aliasLegacyEnv()
+
+async function exists (file) {
+  try { await access(file); return true } catch { return false }
+}
+
+/** Finds kaloko.config.yml (or the former qawalk.config.yml) from `start` upwards and returns {config, root}. */
+export async function loadConfig (start = process.cwd(), explicit) {
+  let file = explicit ? resolve(explicit) : null
+  if (!file) {
+    let dir = resolve(start)
+    while (true) {
+      if (await exists(join(dir, CONFIG_FILE))) { file = join(dir, CONFIG_FILE); break }
+      if (await exists(join(dir, LEGACY_CONFIG_FILE))) { file = join(dir, LEGACY_CONFIG_FILE); break }
+      const parent = dirname(dir)
+      if (parent === dir) break
+      dir = parent
+    }
+  }
+  if (!file) throw new UserError(`${CONFIG_FILE} not found. Run "kaloko init" in the project root.`)
+  // the user-level file holds personal keys and tokens: tests (node --test) and KALOKO_USER_ENV=0 leave it out
+  const userEnv = process.env.KALOKO_USER_ENV !== '0' && !process.env.NODE_TEST_CONTEXT
+  await loadEnvFiles([join(dirname(file), '.env'), ...(userEnv ? [join(homedir(), '.config', 'kaloko', '.env'), join(homedir(), '.config', 'qawalk', '.env')] : [])])
+  aliasLegacyEnv()
+  const config = yaml.load(await readFile(file, 'utf8'))
+  const problems = schemaErrors('config.v1', config)
+  if (problems.length) throw new UserError(`${file} is invalid:\n  - ${problems.join('\n  - ')}`)
+  // projects started before the rename keep their run folder (it is in their .gitignore)
+  config.output ||= !(await exists(join(dirname(file), 'tmp', 'kaloko'))) && (await exists(join(dirname(file), 'tmp', 'qawalk'))) ? 'tmp/qawalk' : 'tmp/kaloko'
+  config.browser ||= {}
+  config.browser.port ||= 9333
+  return { config, root: dirname(file), file }
+}
+
+export function environment (config, name, vars = process.env) {
+  const env = config.environments[name]
+  if (!env) throw new UserError(`Unknown environment "${name}". Known: ${Object.keys(config.environments).join(', ')}`)
+  // a design environment has no application: the CLI serves its folder on 127.0.0.1 while walking (#42)
+  if (env.kind === 'design') return { name, readonly: false, may_create_data: false, blocked_paths: [], allowed_post_paths: [], ...env, kind: 'design', base_url: 'http://127.0.0.1' }
+  // `${VAR}` in base_url: review apps and preview deployments get their URL from CI
+  const expand = (url) => url.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, v) => {
+    if (!vars[v]) throw new UserError(`Environment ${name}: base_url needs ${v} (set it in the environment or .env).`)
+    return vars[v]
+  })
+  // app environments may have no base URL (deep links are app schemes)
+  const base_url = env.base_url == null ? null : typeof env.base_url === 'string' ? expand(env.base_url) : Object.fromEntries(Object.entries(env.base_url).map(([k, v]) => [k, expand(v)]))
+  return { name, readonly: false, may_create_data: true, blocked_paths: [], allowed_post_paths: [], ...env, base_url, ...(name === 'production' ? { readonly: true, may_create_data: false } : {}) }
+}
+
+/** Origins of the environment's base URL (every locale). */
+export function baseOrigins (env) {
+  if (!env.base_url) return []
+  return [...new Set(Object.values(typeof env.base_url === 'string' ? { d: env.base_url } : env.base_url).map((u) => new URL(u).origin))]
+}
+
+export function baseUrl (env, locale) {
+  return env.base_url ? localized(env.base_url, locale).replace(/\/$/, '') : ''
+}
+
+export function urlFor (env, node, locale) {
+  // paths never fall back to another locale's path: a missing locale means the step does not exist there
+  const raw = typeof node.path === 'string' ? node.path : (node.path?.[locale] ?? node.path?.default)
+  if (!raw) return null
+  // `{random}` makes a path that cannot exist (error pages: /no-such-page-{random} must answer 404)
+  const path = raw.replaceAll('{random}', () => Math.random().toString(36).slice(2, 10))
+  if (/^https?:\/\//.test(path)) return path
+  return baseUrl(env, locale) + (path.startsWith('/') ? path : `/${path}`)
+}
+
+/**
+ * Readonly policy for a request. Returns null when allowed, otherwise a reason.
+ * Blocked paths are always blocked; readonly additionally blocks non-GET/HEAD
+ * requests to the environment's hosts unless the path is explicitly allowed.
+ */
+export function policyViolation (env, { method, url }) {
+  let parsed
+  try { parsed = new URL(url) } catch { return null }
+  const path = parsed.pathname
+  const hosts = new Set(Object.values(typeof env.base_url === 'string' ? { d: env.base_url } : env.base_url || {}).map((u) => new URL(u).host))
+  if (!hosts.has(parsed.host)) return null
+  if (env.blocked_paths.some((g) => globToRegExp(g).test(path))) return `path ${path} is blocked in environment ${env.name}`
+  if (env.readonly && !['GET', 'HEAD', 'OPTIONS'].includes(method.toUpperCase())) {
+    if (!env.allowed_post_paths.some((g) => globToRegExp(g).test(path))) return `${method} ${path} is not allowed in readonly environment ${env.name}`
+  }
+  return null
+}
+
+/**
+ * Loads KEY=value lines from env files into process.env without overriding
+ * variables that are already set. Project .env wins over the global file.
+ */
+export async function loadEnvFiles (files) {
+  for (const file of files) {
+    let text
+    try { text = await readFile(file, 'utf8') } catch { continue }
+    for (const line of text.split('\n')) {
+      const m = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/)
+      if (!m || process.env[m[1]] !== undefined) continue
+      process.env[m[1]] = m[2].replace(/^(['"])(.*)\1$/, '$2')
+    }
+  }
+}

package/src/lib/console.mjs

@@ -0,0 +1,13 @@
+// In-memory collection for browsers that are not managed by the daemon.
+const logs = new WeakMap()
+
+export function observeConsole (page) {
+  if (logs.has(page)) return
+  const entries = []
+  logs.set(page, entries)
+  const record = (text, src = null) => entries.push({ ts: new Date().toISOString(), url: page.url(), type: 'console', text, src })
+  page.on('console', (message) => { if (message.type() === 'error') record(message.text(), message.location()?.url || null) })
+  page.on('pageerror', (error) => record(error.message))
+}
+
+export const consoleEntriesFor = (page) => logs.get(page) ?? null

package/src/lib/coverage.mjs

@@ -0,0 +1,75 @@
+// Coverage between the acceptance plan (ACCEPTANCE.md) and the scenario, and flakiness across runs.
+
+/**
+ * Reads the plan template shape: `### <step-id> — <title> (...)` headings and `- AC<n>: text` lines.
+ * @returns {{ steps: Record<string, { title: string, criteria: Record<string, string> }> }}
+ */
+export function parsePlan (markdown) {
+  const steps = {}
+  let current = null
+  for (const raw of String(markdown || '').split('\n')) {
+    const h = raw.match(/^###\s+`?([A-Za-z0-9._-]+)`?\s*(?:—|-|–)\s*(.*?)\s*(?:\(`?[a-z|` ]+`?\))?\s*$/)
+    if (h) { current = h[1]; steps[current] = { title: h[2].trim(), criteria: {} }; continue }
+    if (/^##\s/.test(raw)) { current = null; continue }
+    const ac = raw.match(/^\s*[-*]\s+\*{0,2}(AC\d+)\*{0,2}\s*[:.]\s*(.*)$/)
+    if (ac && current) steps[current].criteria[ac[1]] = ac[2].trim()
+  }
+  return { steps }
+}
+
+/** Criteria planned but not checked, checks without a planned criterion, and steps only on one side. */
+export function coverage (scenario, plan) {
+  const missing = []; const extra = []
+  const scenarioSteps = new Map((scenario.nodes || []).filter((n) => n.kind !== 'decision').map((n) => [n.id, n]))
+  for (const [stepId, step] of Object.entries(plan.steps)) {
+    const node = scenarioSteps.get(stepId)
+    for (const [acId, text] of Object.entries(step.criteria)) {
+      if (!node || !(node.criteria || []).some((c) => c.id === acId)) missing.push({ step: stepId, id: acId, text })
+    }
+  }
+  for (const [stepId, node] of scenarioSteps) {
+    for (const c of node.criteria || []) if (!c.pack && !plan.steps[stepId]?.criteria[c.id]) extra.push({ step: stepId, id: c.id, text: c.text })
+  }
+  const planned = Object.values(plan.steps).reduce((n, s) => n + Object.keys(s.criteria).length, 0)
+  const checked = planned - missing.length
+  return {
+    planned, checked, missing, extra,
+    steps_missing: Object.keys(plan.steps).filter((id) => !scenarioSteps.has(id)),
+    steps_extra: [...scenarioSteps.keys()].filter((id) => !plan.steps[id])
+  }
+}
+
+export function formatCoverage (file, cov) {
+  const lines = [`${file}: ${cov.checked}/${cov.planned} planned criteria have a check${cov.extra.length ? `, ${cov.extra.length} check(s) without a planned criterion` : ''}`]
+  for (const m of cov.missing) lines.push(`  ✕ ${m.step} ${m.id} — planned, no check: ${m.text}`)
+  for (const e of cov.extra) lines.push(`  • ${e.step} ${e.id} — checked, not in the plan: ${e.text}`)
+  if (cov.steps_missing.length) lines.push(`  ✕ steps in the plan without a node: ${cov.steps_missing.join(', ')}`)
+  if (cov.steps_extra.length) lines.push(`  • nodes not in the plan: ${cov.steps_extra.join(', ')}`)
+  return lines.join('\n')
+}
+
+/**
+ * Flaky criteria: verdict differs between runs of the same scenario on the same commit.
+ * @param {object} run the current manifest (nodes with criteria.verdict, git.commit)
+ * @param {object[]} others manifests of other runs of the scenario
+ * @returns {Set<string>} keys `${nodeId}|${criterionId}`
+ */
+export function flaky (run, others) {
+  const out = new Set()
+  const commit = run.git?.commit
+  if (!commit) return out
+  const same = others.filter((r) => r.git?.commit === commit && r.environment?.name === run.environment?.name)
+  if (!same.length) return out
+  for (const n of run.nodes || []) {
+    for (const c of n.criteria || []) {
+      if (!c.verdict || c.verdict === 'not_reached') continue
+      for (const r of same) {
+        const oc = r.nodes?.find((x) => x.id === n.id)?.criteria?.find((x) => x.id === c.id)
+        // a criterion whose check changed between the runs is a scenario edit, not flakiness
+        const sameCheck = !oc?.check || !c.check || JSON.stringify(oc.check) === JSON.stringify(c.check)
+        if (oc?.verdict && oc.verdict !== 'not_reached' && oc.verdict !== c.verdict && sameCheck) { out.add(`${n.id}|${c.id}`); break }
+      }
+    }
+  }
+  return out
+}