kaloko 0.3.0

package/src/lib/evaluate.mjs

@@ -0,0 +1,222 @@
+import { join, resolve } from 'node:path'
+import { readJson, writeJson, sha256 } from './util.mjs'
+import { readCaptures, runPaths } from './run.mjs'
+import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES } from './jev.mjs'
+import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
+import { packHelp } from './pack-help.mjs'
+
+/** Latest capture per node/locale/viewport. */
+export function latestCaptures (entries) {
+  const map = new Map()
+  for (const e of entries) if (e.type === 'capture') map.set(`${e.node}|${e.item || ''}|${e.locale}|${e.viewport}`, e)
+  return [...map.values()]
+}
+
+export const criteriaHash = (node) => sha256(node.criteria || [])
+/** Folder of a capture under steps/: the node, or steps/<node>/items/<item> for a page of a stack step. */
+export const stepIdOf = (e) => (e.item ? `${e.node}/items/${e.item}` : e.node)
+export const evaluationSource = (node, capture) => sha256({ capture, criteria: node.criteria || [], purpose: node.purpose || '' })
+
+/** Criterion verdicts for one capture, combining deterministic results and JEV answers. */
+export function combine ({ node, capture, jev, thresholds }) {
+  return (node.criteria || []).map((c) => {
+    if (c.check.type === 'deterministic') {
+      const r = capture.checks?.[c.id]
+      const verdict = r == null || r.ok == null ? 'not_evaluated' : r.ok ? 'pass' : 'fail'
+      const live = c.check.assert === 'no_console_errors' || c.check.assert === 'no_overflow' || c.check.assert === 'http_status' || /^perf_/.test(c.check.assert || '')
+      return { id: c.id, type: 'deterministic', verdict, evidence: r?.evidence ?? null, ...(capture.rechecked_at && !live ? { rechecked: true } : {}) }
+    }
+    if (c.check.type === 'semantic' && c.check.about === 'console' && !(capture.console_errors || []).length && capture.checks) {
+      return { id: c.id, type: 'semantic', verdict: 'pass', evidence: 'no console errors', score: null, provider: null, note: null }
+    }
+    if (c.check.type === 'semantic') {
+      const answer = jev?.response?.answers?.[c.id]
+      const score = answer?.noul ?? null
+      return { id: c.id, type: 'semantic', verdict: verdictFor(score, thresholds), evidence: null, score, provider: answer ? 'typesafe-jev' : null, note: jev?.skipped || jev?.error || null }
+    }
+    return { id: c.id, type: 'manual', verdict: 'not_evaluated', evidence: c.check.note || null }
+  })
+}
+
+const BATCH = 50
+
+export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
+  const settings = { ...DEFAULTS, ...(config.evaluators?.jev || {}), thresholds: { ...DEFAULTS.thresholds, ...(config.evaluators?.jev?.thresholds || {}) } }
+  const cacheDir = join(resolve(root, config.output), '.cache', 'jev')
+  const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
+  const paths = runPaths(run.dir)
+  const report = { evaluated: 0, semantic_calls: 0, semantic_cached: 0, skipped: [], model: settings.model, drift: 0, outdated: new Set(), via: {}, hosted: null, hostedStop: null, findings: [] }
+  // checks against the design reference and the token set (pack design, #42) need the service and a browser
+  const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
+  try {
+  const latest = latestCaptures(await readCaptures(run.dir))
+  // stack steps: semantic criteria only on the first N pages (items.semantic_sample, default 20); cost stays bounded
+  const sampled = new Map()
+  for (const e of latest) {
+    if (!e.item) continue
+    const node = nodes.get(e.node); if (!node) continue
+    const n = node.items?.semantic_sample ?? 20
+    const set = sampled.get(e.node) || new Set(); sampled.set(e.node, set)
+    if (set.size < n) set.add(e.item)
+  }
+  // in batches of 50 captures: memory holds one batch of page states, not a whole stack of thousands
+  const hosted = hostedSender(config, report)
+  for (let at = 0; at < latest.length; at += BATCH) {
+    const batch = latest.slice(at, at + BATCH)
+    // 1. what each capture needs; 2. the evaluator calls, several at a time; 3. drift and verdicts in order
+    const work = []
+    for (const entry of batch) {
+      const node = nodes.get(entry.node)
+      if (!node) continue
+      const capture = await readJson(join(run.dir, entry.file))
+      // Old captures remain readable, but checks cannot be reinterpreted after
+      // their definitions change. Recapture to bind them to the current criteria.
+      const current = capture.criteria_hash === criteriaHash(node)
+      // console triage needs the evaluator only when the capture has console errors
+      const outOfSample = entry.item && !sampled.get(entry.node)?.has(entry.item)
+      const semantic = outOfSample ? [] : (node.criteria || []).filter((c) => c.check.type === 'semantic' && !(c.check.about === 'console' && !(capture.console_errors || []).length))
+      // quick signals ride along on every captured page state (also steps without semantic criteria)
+      const signals = settings.signals !== false && !outOfSample && !!capture.state
+      const request = (semantic.length || signals) && current ? buildRequest({ node, capture, criteria: semantic, model: settings.model, signals }) : null
+      work.push({ entry, node, capture, current, outOfSample, request, asks: semantic.length > 0, jev: current ? null : { skipped: 'Capture has no matching criteria fingerprint; capture this step again.' } })
+      if (!current) report.outdated.add(entry.node)
+    }
+
+    const queue = work.filter((w) => w.request)
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 6, queue.length)) }, async () => {
+      for (let w = queue.shift(); w; w = queue.shift()) {
+        w.jev = report.hostedStop && !process.env[settings.api_key_env] ? { skipped: report.hostedStop } : await evaluateWithJev({ request: w.request, settings, cacheDir, fetchImpl, hosted })
+        if (w.jev.response) { w.jev.cached ? report.semantic_cached++ : report.semantic_calls++; report.via[w.jev.via] = (report.via[w.jev.via] || 0) + 1; if (w.jev.response.qawalk) report.hosted = w.jev.response.qawalk }
+        // signals alone are best effort: only captures with semantic criteria report why they were not evaluated
+        if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
+      }
+    }))
+    for (const { entry, node, capture, current, outOfSample, request, jev } of work) {
+      const source = evaluationSource(node, capture)
+      if (drift && current && needsDrift(node)) { capture.checks = { ...capture.checks, ...await driftForCapture({ node, entry, capture, runDir: run.dir, ctx: drift }) }; report.drift++ }
+      let criteria = combine({ node, capture: current ? capture : {}, jev, thresholds: settings.thresholds })
+      // pages outside the semantic sample carry only the deterministic results
+      if (outOfSample) criteria = criteria.filter((c) => c.type !== 'semantic' || c.evidence === 'no console errors')
+      const evaluation = {
+        schema: 'qawalk.eval.v1',
+        source,
+        node: entry.node, ...(entry.item ? { item: entry.item, semantic_sample: !outOfSample } : {}), locale: entry.locale, viewport: entry.viewport,
+        evaluated_at: new Date().toISOString(),
+        semantic: jev?.response ? {
+          provider: 'typesafe-jev',
+          via: jev.via || 'own-key',
+          model: jev.response.model || settings.model,
+          prompt_version: PROMPT_VERSION,
+          state_hash: jev.key,
+          cached: jev.cached,
+          usage: jev.response.usage || null,
+          answers: jev.response.answers
+        } : (jev ? { provider: 'typesafe-jev', skipped: jev.skipped || null, error: jev.error || null } : null),
+        thresholds: settings.thresholds,
+        state_chars: request ? JSON.stringify(request.state).length : null,
+        ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
+        criteria
+      }
+      await writeJson(paths.step(stepIdOf(entry), entry.locale, entry.viewport, 'eval.json'), evaluation)
+      report.evaluated++
+      // what did not pass, for the walk summary and agents: step, locale, viewport, criterion and why
+      for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes` : c.note) || null, ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
+    }
+  }
+  } finally { if (drift?.browser) await drift.browser.close() }
+  const { hostedStop, ...out } = report
+  return { ...out, outdated: [...report.outdated] }
+}
+
+/**
+ * While a walk runs: the semantic questions of each new capture go to the evaluator in the background, so the
+ * answers are cached by the time the walk ends and `evaluateRun` only assembles verdicts. Pages of stack steps
+ * are left to evaluateRun (it samples them).
+ */
+/** The quick signals of an answer: { purpose, language, broken } as probabilities of "yes". */
+function signalsOf (jev) {
+  const a = jev?.response?.answers
+  if (!a) return null
+  const out = {}
+  for (const [id, name] of Object.entries(SIGNAL_NAMES)) if (a[id]?.noul != null) out[name] = a[id].noul
+  return Object.keys(out).length ? out : null
+}
+
+export function semanticPrefetch ({ scenario, config, root }) {
+  const settings = { ...DEFAULTS, ...(config.evaluators?.jev || {}), thresholds: { ...DEFAULTS.thresholds, ...(config.evaluators?.jev?.thresholds || {}) } }
+  const cacheDir = join(resolve(root, config.output), '.cache', 'jev')
+  const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
+  const report = { hostedStop: null }
+  const hosted = hostedSender(config, report)
+  const queue = []; const running = new Set(); let asked = 0
+  let idle = null // resolves when the queue drains, for done()
+  const pump = () => {
+    while (running.size < (settings.concurrency || 6) && queue.length) {
+      const job = queue.shift()
+      const p = job().catch(() => {}).finally(() => { running.delete(p); pump() })
+      running.add(p)
+    }
+    if (!running.size && !queue.length && idle) { idle.resolve(); idle = null }
+  }
+  return {
+    add (entries, runDir) {
+      if (config.evaluators?.jev === false || config.evaluators?.jev === null) return
+      for (const e of [].concat(entries || [])) {
+        if (e?.type !== 'capture' || e.item) continue
+        const node = nodes.get(e.node)
+        const semantic = (node?.criteria || []).filter((c) => c.check.type === 'semantic')
+        if (!semantic.length && settings.signals === false) continue
+        queue.push(async () => {
+          const capture = await readJson(join(runDir, e.file))
+          const crit = semantic.filter((c) => !(c.check.about === 'console' && !(capture.console_errors || []).length))
+          const signals = settings.signals !== false && !!capture.state
+          if ((!crit.length && !signals) || capture.criteria_hash !== criteriaHash(node)) return
+          asked++
+          await evaluateWithJev({ request: buildRequest({ node, capture, criteria: crit, model: settings.model, signals }), settings, cacheDir, hosted })
+        })
+      }
+      pump()
+    },
+    async done () {
+      if (running.size || queue.length) { let resolve; const p = new Promise((r) => { resolve = r }); idle = { resolve }; await p }
+      return { asked, stopped: report.hostedStop }
+    }
+  }
+}
+
+/**
+ * Sends a JEV request through the organization's service (hosted evaluation) when a service token is set.
+ * An allowance used up or a service without hosting stops further hosted calls for this run with one message.
+ */
+function hostedSender (config, report) {
+  let client = null
+  return async (body) => {
+    if (report.hostedStop) return { skipped: report.hostedStop }
+    try { if (!client) { const { service } = await import('./service.mjs'); client = service(config) } } catch { return { skipped: 'no evaluator key (TYPESAFE_API_KEY) and no service token (KALOKO_TOKEN) for hosted evaluation' } }
+    try { return { response: await client.request('POST', `/orgs/${config.org}/semantic`, JSON.parse(body)) } } catch (e) {
+      const m = String(e.message || e)
+      if (/HTTP (402|403|503)/.test(m)) { report.hostedStop = m.replace(/^.*HTTP \d+: /, ''); return { skipped: report.hostedStop } }
+      return { error: m }
+    }
+  }
+}
+
+/** Context of the design checks: service client, token set, reference cache and a browser for pixel diffs. */
+async function driftContext ({ run, scenario, config, root }) {
+  const isDesign = run.meta.environment.kind === 'design'
+  const ctx = { org: config.org, scenarioId: scenario.id, isDesign, git: run.meta.git?.branch || null, references: new Map(), cacheDir: driftCacheDir(root, config.output), mode: null, tokens: null, client: null, page: null, browser: null }
+  try { const { service } = await import('./service.mjs'); ctx.client = service(config) } catch (e) { ctx.referenceError = `Design checks need the service: ${e.message}` }
+  let designDir = null
+  try {
+    const { environment } = await import('./config.mjs')
+    const d = Object.keys(config.environments).map((n) => environment(config, n)).find((e) => e.kind === 'design')
+    if (d) { designDir = resolve(root, d.serve); ctx.mode = d.tokens_mode || null }
+  } catch {}
+  ctx.tokens = await loadTokens({ client: ctx.client, config, root, designDir })
+  if (!isDesign && scenario.nodes.some((n) => (n.criteria || []).some((c) => c.check.assert === 'design_pixels'))) {
+    const { launchEphemeral } = await import('./browser.mjs')
+    ctx.browser = await launchEphemeral(config)
+    ctx.page = await (await ctx.browser.newContext()).newPage()
+  }
+  return ctx
+}

package/src/lib/git.mjs

@@ -0,0 +1,70 @@
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+
+const run = promisify(execFile)
+
+async function out (cmd, args, cwd) {
+  try {
+    const { stdout } = await run(cmd, args, { cwd, timeout: 15000 })
+    return stdout.trim()
+  } catch {
+    return null
+  }
+}
+
+/** owner/name of the origin remote, for GitHub and GitLab style URLs; null when absent. */
+export async function repoSlug (cwd) {
+  const url = await out('git', ['remote', 'get-url', 'origin'], cwd)
+  if (!url) return null
+  const m = url.match(/[:/]([^/:]+\/[^/]+?)(?:\.git)?$/)
+  return m ? { slug: m[1], host: url.includes('github.com') ? 'github' : url.includes('gitlab') ? 'gitlab' : 'other' } : null
+}
+
+/**
+ * Project = config.project, else the repository name. Issue = scenario.issue,
+ * else a number found in the scenario id (TASK-1273) or the branch (feat/1273-…).
+ */
+export async function projectAndIssue ({ cwd, config, scenario, branch }) {
+  const repo = await repoSlug(cwd)
+  // the scenario may sort itself into a project; otherwise the config, then the repository name
+  const project = scenario.project
+    ? { slug: scenario.project, repo: repo?.slug || null, source: 'scenario' }
+    : config.project
+      ? { slug: config.project, repo: repo?.slug || null, source: 'config' }
+      : repo ? { slug: repo.slug.split('/').pop(), repo: repo.slug, source: 'git' } : { slug: cwd.split('/').pop(), repo: null, source: 'directory' }
+  let issue = null
+  const given = scenario.issue
+  const num = given?.number || Number((scenario.id.match(/(?:^|[-_#])(\d{2,6})(?:$|[-_])/) || [])[1]) || Number((String(branch || '').match(/(?:^|[/-])(\d{2,6})(?:-|$)/) || [])[1]) || null
+  if (num) {
+    const slug = given?.repo || repo?.slug || null
+    const url = given?.url || (slug && repo?.host === 'github' ? `https://github.com/${slug}/issues/${num}` : slug && repo?.host === 'gitlab' ? `https://gitlab.com/${slug}/-/issues/${num}` : null)
+    issue = { repo: slug, number: num, url, title: given?.title || null }
+  }
+  return { project, issue }
+}
+
+/** Branch, commit, author and open/merged PR (via gh, optional). */
+/** Application version for documentation captures: the tag on HEAD (git describe --exact-match), else null. */
+export async function appVersion (cwd) {
+  return (await out('git', ['describe', '--tags', '--exact-match', 'HEAD'], cwd)) || null
+}
+
+export async function gitMeta (cwd) {
+  const commit = await out('git', ['rev-parse', '--short', 'HEAD'], cwd)
+  if (!commit) return null
+  const [branch, message, author, sha] = await Promise.all([
+    out('git', ['rev-parse', '--abbrev-ref', 'HEAD'], cwd),
+    out('git', ['log', '-1', '--format=%s'], cwd),
+    out('git', ['log', '-1', '--format=%an'], cwd),
+    out('git', ['rev-parse', 'HEAD'], cwd)
+  ])
+  let pr = null
+  const json = await out('gh', ['pr', 'view', '--json', 'number,url,state'], cwd)
+  if (json) {
+    try {
+      const p = JSON.parse(json)
+      pr = { number: p.number, url: p.url, state: String(p.state).toLowerCase() }
+    } catch {}
+  }
+  return { branch, commit, sha, message, author, pr }
+}

package/src/lib/htmlview.mjs

@@ -0,0 +1,130 @@
+// HTML view of captured app pages (#42, experimental): a static copy of the page as it rendered, with its
+// stylesheets, images and fonts rehosted, every script and navigation removed, and secrets and personal data
+// scrubbed. It opens in the same frame as live design steps, so the inspector and pins work on the real product.
+// The service sanitises again when serving (defence in depth): nothing here is trusted by it.
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join, dirname } from 'node:path'
+import { sha256Hex, revisionId, bundleProblems } from './design-core.mjs'
+
+const ASSET_TYPES = { 'text/css': 'css', 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/gif': 'gif', 'image/svg+xml': 'svg', 'image/avif': 'avif', 'font/woff2': 'woff2', 'font/woff': 'woff', 'application/font-woff2': 'woff2', 'application/font-woff': 'woff' }
+export const VIEW_LIMITS = { max_assets: 150, max_asset_bytes: 3 * 1024 * 1024, max_total_bytes: 25 * 1024 * 1024 }
+
+/** Runs in the page: a scrubbed clone of the document with asset URLs replaced by placeholders. Self-contained. */
+export function viewInPage ({ mask = [] }) {
+  const doc = document.documentElement.cloneNode(true)
+  const liveEls = [...document.querySelectorAll('input, textarea, select')]
+  const cloneEls = [...doc.querySelectorAll('input, textarea, select')]
+  // form state as rendered (not passwords): values into attributes so the static copy shows them
+  liveEls.forEach((el, i) => {
+    const c = cloneEls[i]; if (!c) return
+    if (el.type === 'password' || el.type === 'hidden') { c.removeAttribute('value'); return }
+    if (el.tagName === 'TEXTAREA') c.textContent = el.value
+    else if (el.tagName === 'SELECT') [...c.options].forEach((o, k) => { if (el.options[k]?.selected) o.setAttribute('selected', ''); else o.removeAttribute('selected') })
+    else if (el.type === 'checkbox' || el.type === 'radio') { if (el.checked) c.setAttribute('checked', ''); else c.removeAttribute('checked') } else c.setAttribute('value', el.value)
+  })
+  const drop = 'script, noscript, iframe, frame, frameset, object, embed, applet, base, portal, template, link[rel~=manifest], link[rel~=serviceworker], link[rel~=modulepreload], link[rel~=preload][as=script], link[rel~=prefetch], link[rel~=dns-prefetch], link[rel~=preconnect], meta[http-equiv]'
+  doc.querySelectorAll(drop).forEach((e) => e.remove())
+  // inline JSON state (framework hydration data) and csrf metas carry session data
+  doc.querySelectorAll('meta[name*=csrf i], meta[name*=token i], meta[name*=nonce i]').forEach((e) => e.remove())
+  const SECRET_SRC = '\\b(eyJ[\\w-]{10,}\\.[\\w-]{10,}\\.[\\w-]{10,}|(?:sk|pk|rk|ghp|gho|xox[abp])_[A-Za-z0-9_-]{16,}|[A-Za-z0-9+/_-]{40,}={0,2})\\b'
+  // a fresh expression per use: a shared global one would skip matches through lastIndex
+  const SECRET = { test: (v) => new RegExp(SECRET_SRC).test(v), all: () => new RegExp(SECRET_SRC, 'g') }
+  const URL_ATTR = /^(href|src|action|formaction|xlink:href|poster|data|background|srcset|ping)$/i
+  for (const el of doc.querySelectorAll('*')) {
+    for (const a of [...el.attributes]) {
+      if (/^on/i.test(a.name)) { el.removeAttribute(a.name); continue }
+      if (URL_ATTR.test(a.name) && /^\s*(javascript|vbscript|data:(text\/html|application|image\/svg))/i.test(a.value)) { el.removeAttribute(a.name); continue }
+      if (/^(data-|aria-label|title|alt|value|content)/i.test(a.name) && SECRET.test(a.value)) el.setAttribute(a.name, a.value.replace(SECRET.all(), '<token>'))
+    }
+  }
+  doc.querySelectorAll('form').forEach((f) => { f.removeAttribute('action'); f.removeAttribute('method') })
+  // a view is for looking: links keep their target for reference but navigate nowhere
+  doc.querySelectorAll('a[href], area[href]').forEach((a) => { a.setAttribute('data-href', a.getAttribute('href')); a.setAttribute('href', '#') })
+  // personal data: masked text (not only blurred pixels)
+  for (const sel of mask) { try { doc.querySelectorAll(sel).forEach((el) => { el.textContent = '••••'; el.removeAttribute('value') }) } catch {} }
+  const walker = document.createTreeWalker(doc, NodeFilter.SHOW_TEXT)
+  for (let n = walker.nextNode(); n; n = walker.nextNode()) if (SECRET.test(n.textContent)) n.textContent = n.textContent.replace(SECRET.all(), '<token>')
+  // assets → placeholders
+  const assets = []
+  const slot = (url) => { try { const abs = new URL(url, document.baseURI).href; if (!/^https?:/.test(abs)) return null; let i = assets.indexOf(abs); if (i < 0) { i = assets.push(abs) - 1 } return `__QW_ASSET_${i}__` } catch { return null } }
+  doc.querySelectorAll('link[rel~=stylesheet][href], link[rel~=icon][href]').forEach((l) => { const s = slot(l.getAttribute('href')); if (s) l.setAttribute('href', s); else l.remove() })
+  doc.querySelectorAll('link:not([rel~=stylesheet]):not([rel~=icon])').forEach((l) => l.remove())
+  doc.querySelectorAll('img[src], source[src], video[poster], image, use').forEach((el) => {
+    for (const at of ['src', 'poster', 'href', 'xlink:href']) { const v = el.getAttribute(at); if (!v || v.startsWith('data:image/') || v.startsWith('#')) continue; const s = slot(v); if (s) el.setAttribute(at, s); else el.removeAttribute(at) }
+  })
+  doc.querySelectorAll('img[srcset], source[srcset]').forEach((el) => {
+    // the candidate the browser picked is enough for a static copy
+    const cur = el.currentSrc || el.getAttribute('srcset').split(',')[0].trim().split(/\s+/)[0]
+    el.removeAttribute('srcset'); const s = slot(cur); if (s && el.tagName === 'IMG') el.setAttribute('src', s)
+  })
+  const cssUrls = (text) => text.replace(/url\(\s*(['"]?)([^'")]+)\1\s*\)/gi, (m, q, u) => u.startsWith('data:') ? m : (slot(u) ? `url(${slot(u)})` : 'none'))
+  doc.querySelectorAll('style').forEach((s) => { s.textContent = cssUrls(s.textContent) })
+  doc.querySelectorAll('[style]').forEach((el) => el.setAttribute('style', cssUrls(el.getAttribute('style'))))
+  return { html: '<!doctype html>\n' + doc.outerHTML, assets, base: document.baseURI }
+}
+
+/**
+ * Builds the HTML view of the page for one capture: downloads the assets through the browser context (same
+ * cookies and gate headers as the capture), rewrites CSS url()s, writes the bundle into the run directory
+ * (live/<revision>/…) and returns the revision ({ id, entry, files, sizes }).
+ */
+export async function buildHtmlView ({ page, run, scenarioId, node, locale, viewport, mask = [] }) {
+  const { html, assets } = await page.evaluate(viewInPage, { mask })
+  const request = page.context().request
+  const files = {}, sizes = {}, bodies = new Map()
+  let total = 0
+  const names = new Map()
+  const fetchAsset = async (url, depth = 0) => {
+    if (names.has(url)) return names.get(url)
+    if (names.size >= VIEW_LIMITS.max_assets) return null
+    names.set(url, null)
+    let res
+    try { res = await request.get(url, { timeout: 15000, failOnStatusCode: false, maxRedirects: 3 }) } catch { return null }
+    if (!res.ok()) return null
+    const type = (res.headers()['content-type'] || '').split(';')[0].trim().toLowerCase()
+    const ext = ASSET_TYPES[type]
+    if (!ext) return null
+    let body = await res.body()
+    if (body.length > VIEW_LIMITS.max_asset_bytes || total + body.length > VIEW_LIMITS.max_total_bytes) return null
+    if (ext === 'css' && depth < 2) {
+      // stylesheets point at fonts, images and other sheets relative to themselves: rehost those too
+      let css = body.toString('utf8')
+      const refs = [...css.matchAll(/url\(\s*(['"]?)([^'")]+)\1\s*\)|@import\s+(['"])([^'"]+)\3/gi)].map((m) => m[2] || m[4]).filter((u) => u && !u.startsWith('data:'))
+      for (const ref of [...new Set(refs)]) {
+        let abs; try { abs = new URL(ref, url).href } catch { continue }
+        const name = await fetchAsset(abs, depth + 1)
+        css = css.split(ref).join(name ? name.replace('assets/', '') : 'about:blank')
+      }
+      body = Buffer.from(css.replace(/expression\s*\(|javascript:|behavior\s*:/gi, ''), 'utf8')
+    }
+    const hash = await sha256Hex(body)
+    const name = `assets/${hash.slice(0, 16)}.${ext}`
+    total += body.length
+    names.set(url, name)
+    files[name] = hash; sizes[name] = body.length; bodies.set(name, body)
+    return name
+  }
+  let out = html
+  for (const [i, url] of assets.entries()) {
+    const name = await fetchAsset(url)
+    out = out.split(`__QW_ASSET_${i}__`).join(name || 'about:blank')
+  }
+  const htmlBody = Buffer.from(out, 'utf8')
+  files['index.html'] = await sha256Hex(htmlBody); sizes['index.html'] = htmlBody.length; bodies.set('index.html', htmlBody)
+  const problems = bundleProblems(Object.keys(files).map((p) => ({ path: p, bytes: sizes[p] })), 'index.html')
+  if (problems.length) throw new Error(`HTML view of ${node} (${locale}, ${viewport}): ${problems.join('; ')}`)
+  const state = `view:${viewport}`
+  const id = await revisionId({ scenario: scenarioId, node, locale, state, kind: 'html_view', entry: 'index.html', files })
+  for (const [p, body] of bodies) { const to = join(run.dir, 'live', id, p); await mkdir(dirname(to), { recursive: true }); await writeFile(to, body) }
+  return { id, node, locale, state, kind: 'html_view', entry: 'index.html', files, sizes }
+}
+
+/** Whether an environment may keep HTML views: opted in, and never production pages behind a sign-in. */
+export function htmlViewAllowed (env, scenario) {
+  if (!env?.html_view) return false
+  if (env.name === 'production' || env.readonly) {
+    const signedIn = Object.keys(env.accounts || {}).length || Object.values(scenario.sessions || {}).some((s) => s.account)
+    if (signedIn) return false
+  }
+  return true
+}

package/src/lib/jev.mjs

@@ -0,0 +1,95 @@
+import { join } from 'node:path'
+import { readJson, writeJson, sha256 } from './util.mjs'
+
+export const PROMPT_VERSION = 'qawalk-jev-v1'
+export const DEFAULTS = { hosted: true, signals: true, concurrency: 6, model: 'jev-1.13.0', base_url: 'https://api.typesafe.ai', api_key_env: 'TYPESAFE_API_KEY', thresholds: { pass: 0.85, fail: 0.15 }, max_state_chars: 90000 }
+
+/**
+ * Quick signals asked beside the criteria of every capture: cheap, independent questions about the page that do
+ * not change verdicts; they point a person at a step that looks wrong. Question ids start with "_" (the service
+ * does not count them against the allowance).
+ */
+export const SIGNALS = {
+  _purpose: (node) => node.purpose ? 'Does `page` do what `screen_purpose` says this screen must achieve?' : null,
+  _language: (node, capture) => capture.locale ? `Is the visible text of \`page\` mainly in the language with code "${capture.locale}"?` : null,
+  _broken: () => 'Does `page` look broken: an error message, placeholder text (lorem ipsum, TODO, {{variables}}), missing content or a blank page?'
+}
+export const SIGNAL_NAMES = { _purpose: 'purpose', _language: 'language', _broken: 'broken' }
+
+/** Builds the JEV request for one capture: one `noul` question per semantic criterion, plus the quick signals. */
+export function buildRequest ({ node, capture, criteria, model, signals = false }) {
+  const scopes = {}
+  const questions = {}
+  for (const c of criteria) {
+    const scoped = c.check.scope && capture.scoped?.[c.id]
+    if (scoped) scopes[c.id] = scoped
+    const target = c.check.about === 'console' ? '`console_errors` (browser console errors captured on this page) together with `screen_purpose`' : scoped ? `\`scopes.${c.id}\` (the part of the page matching ${c.check.scope})` : '`page`'
+    questions[c.id] = {
+      type: 'noul',
+      instructions: `The state describes one web page captured during an acceptance test. \`screen_purpose\` states what the page must achieve. Answer only from ${target}. Question: ${c.check.question}`,
+      ...(c.check.true || c.check.false ? { criteria: { true: c.check.true || 'Yes, the criterion is met.', false: c.check.false || 'No, the criterion is not met.' } } : {})
+    }
+  }
+  if (signals) {
+    for (const [id, q] of Object.entries(SIGNALS)) {
+      const text = q(node, capture)
+      if (text) questions[id] = { type: 'noul', instructions: `The state describes one web page captured during an acceptance test. Answer only from \`page\`. Question: ${text}` }
+    }
+  }
+  const state = {
+    screen_purpose: node.purpose || '',
+    url: capture.url,
+    locale: capture.locale,
+    viewport: capture.viewport,
+    page: capture.state,
+    ...(Object.keys(scopes).length ? { scopes } : {}),
+    ...(criteria.some((c) => c.check.about === 'console') ? { console_errors: (capture.console_errors || []).slice(0, 30) } : {})
+  }
+  return { model, state, questions }
+}
+
+export function verdictFor (noul, thresholds) {
+  if (noul == null) return 'not_evaluated'
+  if (noul >= thresholds.pass) return 'pass'
+  if (noul <= thresholds.fail) return 'fail'
+  return 'needs_review'
+}
+
+/**
+ * Calls JEV with a content-addressed cache. Returns {response, cached, via} or {skipped} / {error}.
+ * With the evaluator key set locally the call goes to TypeSafe directly (via: 'own-key'); without it, and with a
+ * `hosted` sender (the organization's service token), the service evaluates with its key (via: 'qawalk').
+ */
+export async function evaluateWithJev ({ request, settings, cacheDir, fetchImpl = fetch, hosted = null }) {
+  const apiKey = process.env[settings.api_key_env]
+  let body = JSON.stringify(request)
+  if (body.length > settings.max_state_chars + 20000) return { skipped: `state too large (${body.length} chars)` }
+  const key = sha256(body + PROMPT_VERSION)
+  const cacheFile = join(cacheDir, `${key}.json`)
+  const cached = await readJson(cacheFile, null)
+  if (cached) return { response: cached, cached: true, key, via: cached.qawalk ? 'qawalk' : 'own-key' }
+  if (!apiKey && hosted && settings.hosted !== false) {
+    const r = await hosted(body)
+    if (r.skipped || r.error) return r
+    await writeJson(cacheFile, r.response)
+    return { response: r.response, cached: !!r.response.qawalk?.cached, key, via: 'qawalk' }
+  }
+  if (!apiKey) return { skipped: `${settings.api_key_env} is not set${hosted ? '' : ' and the service is not configured (KALOKO_TOKEN)'}` }
+  for (let attempt = 0; attempt < 4; attempt++) {
+    const res = await fetchImpl(`${settings.base_url.replace(/\/$/, '')}/v1/systemone`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+      body
+    })
+    if (res.status === 429 || res.status === 529) {
+      const wait = Number(res.headers.get('retry-after')) * 1000 || 1000 * 2 ** attempt
+      await new Promise((r) => setTimeout(r, wait))
+      continue
+    }
+    if (!res.ok) return { error: `JEV HTTP ${res.status}: ${(await res.text()).slice(0, 300)}` }
+    const response = await res.json()
+    await writeJson(cacheFile, response)
+    return { response, cached: false, key, via: 'own-key' }
+  }
+  return { error: 'JEV rate limited after 4 attempts' }
+}