kaloko 0.4.7 → 0.4.9

package/src/lib/stability.mjs

@@ -0,0 +1,249 @@
+// Auto-detected unstable regions (#20). Two sources:
+//  - repeated screenshots of one capture (`kaloko walk --stability N`): what moves while nobody touches the page
+//    (clocks, carousels, animations) lands in the capture record as `unstable`;
+//  - earlier local runs of the same scenario and environment (`kaloko stability`): what differs on every load
+//    (ads, avatars, "3 minutes ago").
+// `kaloko stability` turns both into ignore suggestions per step (`ignore: [{ selector } | { rect, viewport }]`),
+// written next to the run and, with --apply, into the scenario. Pixel diffs on the canvas and the design pixel
+// check skip ignored regions.
+import { readFile, writeFile, readdir } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import yaml from 'js-yaml'
+import { readJson, writeJson, UserError } from './util.mjs'
+import { readCaptures } from './run.mjs'
+import { latestCaptures } from './evaluate.mjs'
+import { decodePng } from './png.mjs'
+import { unstableRegions, mergeRects, ignoreFor } from './stability-core.mjs'
+
+const MAX_HEIGHT = 12000
+
+/** Copies the top-left width × height of an RGBA image. */
+function cropRgba (img, width, height) {
+  if (img.width === width) return img.data.subarray(0, width * height * 4)
+  const out = new Uint8ClampedArray(width * height * 4)
+  for (let y = 0; y < height; y++) out.set(img.data.subarray(y * img.width * 4, y * img.width * 4 + width * 4), y * width * 4)
+  return out
+}
+
+/** Regions that change while the page sits still: `n` screenshots `delay` ms apart (CSS pixels). */
+export async function repeatRegions (page, { n = 3, delay = 800, fullPage = true } = {}) {
+  const shots = []
+  for (let i = 0; i < n; i++) {
+    if (i) await page.waitForTimeout(delay)
+    shots.push(decodePng(await page.screenshot({ fullPage, scale: 'css', animations: 'allow' })))
+  }
+  const width = Math.min(...shots.map((s) => s.width)); const height = Math.min(MAX_HEIGHT, ...shots.map((s) => s.height))
+  const r = unstableRegions({ images: shots.map((s) => cropRgba(s, width, height)), width, height })
+  return { shots: n, regions: r.regions, ...(r.too_much ? { too_much: Math.round(r.changed * 100) / 100 } : {}) }
+}
+
+/** Ignore regions of one capture as page rectangles (CSS px): selectors resolved in the live page, rects as given. */
+export async function resolveIgnore (page, node, locale, viewport) {
+  const entries = ignoreFor(node, locale, viewport)
+  if (!entries.length) return []
+  const sels = entries.filter((e) => e.selector).map((e) => e.selector)
+  const found = sels.length ? await page.evaluate((list) => list.flatMap((sel) => {
+    let els = []
+    try { els = [...document.querySelectorAll(sel)] } catch { return [] }
+    return els.map((el) => { const r = el.getBoundingClientRect(); return { x: Math.round(r.left + scrollX), y: Math.round(r.top + scrollY), w: Math.round(r.width), h: Math.round(r.height), selector: sel } }).filter((r) => r.w > 0 && r.h > 0).slice(0, 20)
+  }), sels).catch(() => []) : []
+  return [...found, ...entries.filter((e) => e.rect).map((e) => ({ x: e.rect[0], y: e.rect[1], w: e.rect[2], h: e.rect[3] }))]
+}
+
+/** Local runs of the scenario and environment of `run`, newest first (meta and directory). */
+export async function siblingRuns ({ root, config, run, sameCommit = false }) {
+  const base = resolve(root, config.output)
+  let names = []
+  try { names = (await readdir(base, { withFileTypes: true })).filter((e) => e.isDirectory() && !e.name.startsWith('.')).map((e) => e.name) } catch { return [] }
+  const out = []
+  for (const name of names) {
+    const meta = await readJson(join(base, name, 'meta.json'), null)
+    if (!meta || meta.id === run.meta.id || meta.scenario?.id !== run.meta.scenario?.id || meta.environment?.name !== run.meta.environment?.name) continue
+    if (sameCommit && (meta.git?.commit || null) !== (run.meta.git?.commit || null)) continue
+    out.push({ dir: join(base, name), meta })
+  }
+  return out.sort((a, b) => String(b.meta.started_at).localeCompare(String(a.meta.started_at)))
+}
+
+const key = (e) => `${e.node}|${e.locale}|${e.viewport}`
+
+// the self-contained detector, injected into a blank page to compare WebP captures there
+const DETECTOR = `window.__mergeRects = ${mergeRects.toString()}; const mergeRects = window.__mergeRects; window.__unstableRegions = ${unstableRegions.toString()};`
+
+/** Cross-run regions of one step, locale and viewport: images drawn at the page's CSS width. */
+async function crossRegions (page, sources, cssWidth) {
+  return page.evaluate(async ({ sources, W, MAX }) => {
+    const load = (src) => new Promise((ok, no) => { const i = new Image(); i.onload = () => ok(i); i.onerror = no; i.src = src })
+    const imgs = await Promise.all(sources.map(load))
+    const H = Math.min(MAX, ...imgs.map((i) => Math.round(i.naturalHeight * W / i.naturalWidth)))
+    const data = imgs.map((img) => { const c = document.createElement('canvas'); c.width = W; c.height = H; const g = c.getContext('2d', { willReadFrequently: true }); g.drawImage(img, 0, 0, W, Math.round(img.naturalHeight * W / img.naturalWidth)); return g.getImageData(0, 0, W, H).data })
+    return window.__unstableRegions({ images: data, width: W, height: H })
+  }, { sources, W: cssWidth, MAX: MAX_HEIGHT })
+}
+
+const dataUrl = async (file) => `data:image/${file.endsWith('.png') ? 'png' : 'webp'};base64,${(await readFile(file)).toString('base64')}`
+const withBase = (html, url) => {
+  if (!url || /<base\s/i.test(html)) return html
+  const tag = `<base href="${String(url).replace(/"/g, '&quot;')}">`
+  return /<head[^>]*>/i.test(html) ? html.replace(/<head[^>]*>/i, (m) => m + tag) : tag + html
+}
+
+/** A stable CSS selector for the element that covers a region in the stored HTML, or null (runs in the page). */
+function selectorAt (r) {
+  const cx = r.x + r.w / 2; const cy = r.y + r.h / 2
+  window.scrollTo(0, Math.max(0, cy - innerHeight / 2))
+  const hit = document.elementFromPoint(cx - scrollX, cy - scrollY)
+  const fit = (e) => {
+    const b = e.getBoundingClientRect(); const x = b.left + scrollX; const y = b.top + scrollY
+    const ix = Math.max(0, Math.min(x + b.width, r.x + r.w) - Math.max(x, r.x)); const iy = Math.max(0, Math.min(y + b.height, r.y + r.h) - Math.max(y, r.y))
+    return { covers: (ix * iy) / (r.w * r.h), ratio: (b.width * b.height) / (r.w * r.h) }
+  }
+  let best = null
+  for (let e = hit; e && e !== document.body && e !== document.documentElement; e = e.parentElement) {
+    const f = fit(e)
+    if (f.ratio > 6) break
+    if (f.covers >= 0.7) { best = e; break }
+  }
+  if (!best) return null
+  const esc = (s) => CSS.escape(s)
+  const unique = (s) => { try { return document.querySelectorAll(s).length === 1 } catch { return false } }
+  // generated ids and state classes change between builds; leave them out
+  const stableId = (e) => e.id && !/\d{4,}|[a-f0-9]{8,}|^:r/i.test(e.id)
+  const part = (e) => e.tagName.toLowerCase() + [...e.classList].filter((c) => !/\d{4,}|[a-f0-9]{6,}|^(is|has)-|active|current|visible|show|open|selected|swiper-slide-/i.test(c)).slice(0, 2).map((c) => '.' + esc(c)).join('')
+  if (stableId(best) && unique('#' + esc(best.id))) return '#' + esc(best.id)
+  let sel = part(best)
+  if (sel.includes('.') && unique(sel)) return sel
+  for (let e = best.parentElement, d = 0; e && e !== document.body && d < 4; e = e.parentElement, d++) {
+    if (stableId(e) && unique(`#${esc(e.id)} ${sel}`)) return `#${esc(e.id)} ${sel}`
+    sel = `${part(e)} > ${sel}`
+    if (unique(sel)) return sel
+  }
+  return null
+}
+
+/**
+ * Ignore suggestions for the steps of `run` from its own repeated shots and from earlier runs.
+ * @returns {{ runs: string[], steps: Array<{ node, ignore: object[], found: object[] }>, skipped: string[] }}
+ */
+export async function detectUnstable ({ root, config, run, scenario, last = 4, runs = null, sameCommit = false, launch = null }) {
+  const others = runs ? (await Promise.all(runs.map(async (id) => {
+    const dir = id.includes('/') ? resolve(id) : join(resolve(root, config.output), id)
+    const meta = await readJson(join(dir, 'meta.json'), null)
+    if (!meta) throw new UserError(`Run ${id} not found in ${config.output}.`)
+    return { dir, meta }
+  }))) : (await siblingRuns({ root, config, run, sameCommit })).slice(0, Math.max(0, last - 1))
+  const all = [{ dir: run.dir, meta: run.meta }, ...others]
+  const maps = await Promise.all(all.map(async (r) => new Map(latestCaptures(await readCaptures(r.dir)).filter((e) => !e.item).map((e) => [key(e), { ...e, dir: r.dir }]))))
+  const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
+  const found = new Map() // node → [{ locale, viewport, x, y, w, h, share, source }]
+  const skipped = []
+  const browser = await (launch ? launch() : (await import('./browser.mjs')).launchEphemeral(config))
+  try {
+    const page = await browser.newPage()
+    await page.setContent('<!doctype html><title>stability</title>')
+    await page.addScriptTag({ content: DETECTOR })
+    for (const [k, entry] of maps[0]) {
+      if (!nodes.has(entry.node)) continue
+      const caps = []
+      for (const m of maps) { const e = m.get(k); if (e) caps.push({ e, cap: await readJson(join(e.dir, e.file), null) }) }
+      const mine = caps[0].cap
+      if (!mine?.files) continue
+      const regions = []
+      for (const { cap } of caps) for (const r of cap?.unstable?.regions || []) regions.push({ ...r, source: 'repeat' })
+      const usable = caps.filter(({ cap }) => cap?.files?.image || cap?.files?.png)
+      if (usable.length >= 2) {
+        try {
+          const cssWidth = mine.size?.width || mine.size?.viewport?.[0] || 1280
+          const r = await crossRegions(page, await Promise.all(usable.map(({ e, cap }) => dataUrl(join(e.dir, cap.files.image || cap.files.png)))), cssWidth)
+          if (r.too_much) skipped.push(`${entry.node} ${entry.locale}/${entry.viewport}: ${Math.round(r.changed * 100)} % of the screen differs between runs; that is a changed page, not a noisy region`)
+          for (const x of r.regions) regions.push({ ...x, source: 'runs' })
+        } catch (error) { skipped.push(`${entry.node} ${entry.locale}/${entry.viewport}: ${error.message}`) }
+      }
+      if (!regions.length) continue
+      const merged = mergeRects(regions.map(({ x, y, w, h, share }) => ({ x, y, w, h, share })), 8)
+      const list = found.get(entry.node) || []
+      for (const r of merged) list.push({ ...r, locale: entry.locale, viewport: entry.viewport, html: mine.files.html ? join(entry.dir, mine.files.html) : null, url: mine.url, sources: [...new Set(regions.filter((x) => x.x < r.x + r.w && r.x < x.x + x.w && x.y < r.y + r.h && r.y < x.y + x.h).map((x) => x.source))] })
+      found.set(entry.node, list)
+    }
+    // regions → selectors in the stored HTML (scripts off); rectangles when no element fits
+    const steps = []
+    for (const [node, list] of found) {
+      const ignore = []
+      for (const r of list) {
+        let selector = null
+        if (r.html) {
+          const vp = run.meta.scope.viewports?.[r.viewport] || [1280, 800]
+          const ctx = await browser.newContext({ viewport: { width: vp[0], height: vp[1] }, javaScriptEnabled: false })
+          try {
+            const p = await ctx.newPage()
+            await p.setContent(withBase(await readFile(r.html, 'utf8'), r.url), { waitUntil: 'load', timeout: 20000 }).catch(() => {})
+            selector = await p.evaluate(selectorAt, { x: r.x, y: r.y, w: r.w, h: r.h }).catch(() => null)
+          } finally { await ctx.close() }
+        }
+        const note = `${r.sources.includes('repeat') ? 'moves while the page is idle' : 'changes between runs'} (${r.locale}/${r.viewport})`
+        const known = selector && ignore.find((x) => x.selector === selector)
+        if (known) { known.viewports = [...new Set([...(known.viewports || []), r.viewport])]; continue }
+        ignore.push(selector ? { selector, viewports: [r.viewport], note } : { rect: [r.x, r.y, r.w, r.h], viewport: r.viewport, note })
+      }
+      // a selector seen in every viewport of the run needs no viewport list
+      const vps = Object.keys(run.meta.scope.viewports || {})
+      for (const x of ignore) if (x.viewports && vps.every((v) => x.viewports.includes(v))) delete x.viewports
+      steps.push({ node, ignore, found: list.map(({ html, url, ...r }) => r) })
+    }
+    return { runs: all.map((r) => r.meta.id), steps, skipped }
+  } finally { await browser.close() }
+}
+
+/** The suggestions as a YAML fragment per step, ready to paste under each node. */
+export function suggestionsYaml (report) {
+  return report.steps.map((s) => `# ${s.node}\nignore:\n${s.ignore.map((x) => `  - ${yaml.dump(x, { flowLevel: 0, lineWidth: 200 }).trim()}`).join('\n')}`).join('\n\n')
+}
+
+/**
+ * Inserts the suggestions into the scenario file under each node that has no `ignore` yet (block-style nodes only;
+ * comments and formatting stay). Returns { applied: [node], skipped: [reason] }.
+ */
+export async function applySuggestions (file, report) {
+  let text = await readFile(file, 'utf8')
+  const applied = []; const skipped = []
+  const parsed = yaml.load(text)
+  for (const s of report.steps) {
+    const node = (parsed?.nodes || []).find((n) => n.id === s.node)
+    if (!node) { skipped.push(`${s.node}: not in ${file}`); continue }
+    if (node.ignore) { skipped.push(`${s.node}: already has ignore; merge by hand`); continue }
+    const lines = text.split('\n')
+    const at = lines.findIndex((l) => new RegExp(`^(\\s*)- id: ['"]?${s.node.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}['"]?\\s*(#.*)?$`).test(l))
+    if (at < 0) { skipped.push(`${s.node}: node is not in block style; paste the suggestion by hand`); continue }
+    const indent = lines[at].match(/^(\s*)-/)[1] + '  '
+    const block = [`${indent}ignore:`, ...s.ignore.map((x) => `${indent}  - ${yaml.dump(x, { flowLevel: 0, lineWidth: 400 }).trim()}`)]
+    lines.splice(at + 1, 0, ...block)
+    text = lines.join('\n')
+    applied.push(s.node)
+  }
+  if (applied.length) {
+    yaml.load(text) // must still parse
+    await writeFile(file, text)
+  }
+  return { applied, skipped }
+}
+
+/** `kaloko stability`: detect, write stability.json and stability.yml in the run, print, optionally apply. */
+export async function stabilityCmd ({ root, config, run, scenario, args, say }) {
+  const report = await detectUnstable({ root, config, run, scenario, last: Number(args.last) || 4, runs: typeof args.runs === 'string' ? args.runs.split(',').map((s) => s.trim()).filter(Boolean) : null, sameCommit: !!args['same-commit'] })
+  await writeJson(join(run.dir, 'stability.json'), { schema: 'qawalk.stability.v1', generated_at: new Date().toISOString(), ...report })
+  if (args.json) { say(JSON.stringify(report, null, 2)); return report }
+  const repeats = report.steps.some((s) => s.found.some((r) => r.sources.includes('repeat')))
+  say(`Compared ${report.runs.length} run(s)${report.runs.length < 2 && !repeats ? ' (one run and no repeated shots: walk again, or walk with --stability 3, to find regions that move)' : ''}: ${report.steps.length ? `${report.steps.reduce((n, s) => n + s.ignore.length, 0)} unstable region(s) in ${report.steps.length} step(s).` : 'no unstable regions.'}`)
+  for (const s of report.skipped) say(`  ! ${s}`)
+  if (!report.steps.length) return report
+  const text = suggestionsYaml(report)
+  await writeFile(join(run.dir, 'stability.yml'), text + '\n')
+  say(`\n${text}\n\nSuggestions: ${join(run.dir, 'stability.yml')}`)
+  if (args.apply) {
+    const file = resolve(root, run.meta.scenario_file)
+    const r = await applySuggestions(file, report)
+    if (r.applied.length) say(`Added ignore to ${r.applied.join(', ')} in ${run.meta.scenario_file}. Capture those steps again so the canvas and diffs use them.`)
+    for (const x of r.skipped) say(`  skipped ${x}`)
+  } else say('Check them on the screenshots, then run kaloko stability --apply (or paste them into the scenario).')
+  return report
+}

package/src/lib/vision-core.mjs

@@ -0,0 +1,132 @@
+// Vision evaluator (#17), shared by the CLI and the service: which criteria go to it, the request for a
+// vision-capable model (Anthropic Messages API with image input and a strict JSON answer), the parsed answer, and the
+// one-sentence reason every semantic verdict carries. Pure: no Node APIs (the worker imports it).
+//
+// The default text evaluator (JEV) reads a reduced page state and returns a probability only; it takes no images
+// and gives no reasons. This second provider sees the screenshot and explains itself. It is asked only for criteria
+// marked `vision: true` and for text answers in the uncertain band, so the cost stays bounded.
+
+export const VISION_PROMPT_VERSION = 'kaloko-vision-v1'
+export const VISION_DEFAULTS = {
+  provider: 'anthropic', model: 'claude-opus-5-5', api_key_env: 'ANTHROPIC_API_KEY', base_url: 'https://api.anthropic.com',
+  when: 'uncertain', uncertain: [0.15, 0.85], max_calls: 40, max_width: 1000, tile_height: 1400, max_tiles: 2,
+  effort: 'low', reason_language: 'en', hosted: true, concurrency: 3
+}
+/** Models the service accepts for hosted vision questions. */
+export const VISION_MODEL = /^claude-[a-z0-9-]{3,40}$/
+/** One hosted vision question counts as this many questions of the monthly allowance. */
+export const VISION_WEIGHT = 5
+export const MAX_VISION_IMAGES = 5
+// models that take the server-side refusal fallback ("default" routing)
+const FALLBACK_MODELS = new Set(['claude-opus-5-5', 'claude-opus-5', 'claude-fable-5-1', 'claude-sonnet-5-5'])
+
+/** Vision settings from the config (`evaluators.vision`; false or when: off turns it off). */
+export function visionSettings (config) {
+  const v = config?.evaluators?.vision
+  if (v === false || v === null) return null
+  const s = { ...VISION_DEFAULTS, ...(v || {}) }
+  return s.when === 'off' ? null : s
+}
+
+/**
+ * The semantic criteria of one capture that go to the vision evaluator: flagged ones (check.vision) always,
+ * every one with when: always, and with when: uncertain those whose text score lies strictly inside the band.
+ * Console triage stays with the text evaluator.
+ */
+export function visionCriteria ({ criteria, answers = {}, settings }) {
+  if (!settings) return []
+  const [lo, hi] = settings.uncertain || VISION_DEFAULTS.uncertain
+  return criteria.filter((c) => {
+    if (c.check?.type !== 'semantic' || c.check.about === 'console') return false
+    if (c.check.vision === true) return true
+    if (settings.when === 'always') return true
+    if (settings.when !== 'uncertain') return false
+    const s = answers?.[c.id]?.noul
+    return s != null && s > lo && s < hi
+  })
+}
+
+const SYSTEM = 'You check web pages captured during acceptance tests. You see screenshots of one page and answer yes/no questions about it. Judge only what is visible in the images and the stated facts; do not guess about content you cannot see. For each question give the probability (0 to 1) that the answer is yes, and one short sentence that names what in the screenshot decided it.'
+
+/**
+ * The Messages API request body for one capture.
+ * questions: [{ id, question, true?, false?, image? }] (image: label of the crop meant for this question)
+ * images: [{ label, media_type, data }] (base64, at most MAX_VISION_IMAGES)
+ */
+export function buildVisionRequest ({ model, effort = 'low', context = {}, questions, images, reasonLanguage = 'en' }) {
+  const content = []
+  for (const img of images.slice(0, MAX_VISION_IMAGES)) {
+    content.push({ type: 'text', text: `Image "${img.label}"` })
+    content.push({ type: 'image', source: { type: 'base64', media_type: img.media_type, data: img.data } })
+  }
+  const facts = [
+    context.purpose ? `What this screen must achieve: ${context.purpose}` : null,
+    context.url ? `URL: ${context.url}` : null,
+    context.locale ? `Expected language: ${context.locale}` : null,
+    context.viewport ? `Viewport: ${context.viewport}` : null
+  ].filter(Boolean)
+  const list = questions.map((q) => ({
+    id: q.id,
+    question: q.question,
+    ...(q.true ? { counts_as_yes: q.true } : {}),
+    ...(q.false ? { counts_as_no: q.false } : {}),
+    look_at: q.image ? `image "${q.image}" (the part of the page the question is about)` : 'the full-page images'
+  }))
+  content.push({ type: 'text', text: `${facts.join('\n')}\n\nQuestions (JSON):\n${JSON.stringify(list, null, 1)}\n\nAnswer every question once, by id. "reason": one sentence of at most 25 words${reasonLanguage && reasonLanguage !== 'en' ? ` in the language with code "${reasonLanguage}"` : ''}, saying what you saw.` })
+  const body = {
+    model,
+    max_tokens: 4096,
+    system: SYSTEM,
+    messages: [{ role: 'user', content }],
+    output_config: {
+      effort,
+      format: {
+        type: 'json_schema',
+        schema: {
+          type: 'object',
+          properties: { answers: { type: 'array', items: { type: 'object', properties: { id: { type: 'string' }, probability: { type: 'number' }, reason: { type: 'string' } }, required: ['id', 'probability', 'reason'], additionalProperties: false } } },
+          required: ['answers'],
+          additionalProperties: false
+        }
+      }
+    }
+  }
+  // a declined request is re-run on a fallback model in the same call instead of ending without answers
+  if (FALLBACK_MODELS.has(model)) body.fallbacks = 'default'
+  return body
+}
+
+/** Headers of the Messages API call (beta header only with the refusal fallback). */
+export function visionHeaders (apiKey, body) {
+  return { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01', 'content-type': 'application/json', ...(body.fallbacks ? { 'anthropic-beta': 'server-side-fallback-2026-07-01' } : {}) }
+}
+
+const sentence = (s) => { const t = String(s || '').replace(/\s+/g, ' ').trim(); const m = t.match(/^.{10,300}?[.!?](?=\s|$)/); return (m ? m[0] : t).slice(0, 300) }
+
+/** { model, answers: { id: { probability, reason } }, usage } from a Messages API response; throws on refusal. */
+export function parseVisionResponse (resp, ids = null) {
+  if (resp?.stop_reason === 'refusal') throw new Error(`the vision evaluator declined (${resp.stop_details?.category || 'refusal'})`)
+  const text = (resp?.content || []).filter((b) => b.type === 'text').map((b) => b.text).join('')
+  let parsed
+  try { parsed = JSON.parse(text) } catch { throw new Error(`the vision evaluator did not answer in JSON${resp?.stop_reason === 'max_tokens' ? ' (cut off)' : ''}`) }
+  const answers = {}
+  for (const a of parsed?.answers || []) {
+    if (!a || typeof a.id !== 'string' || (ids && !ids.includes(a.id))) continue
+    const p = Number(a.probability)
+    if (!Number.isFinite(p)) continue
+    answers[a.id] = { probability: Math.round(Math.min(1, Math.max(0, p)) * 1000) / 1000, reason: sentence(a.reason) }
+  }
+  return { model: resp.model || null, answers, usage: resp.usage ? { input_tokens: resp.usage.input_tokens ?? null, output_tokens: resp.usage.output_tokens ?? null } : null }
+}
+
+/**
+ * The one sentence stored with a semantic verdict when no model explained it (the text evaluator returns a
+ * probability only): what was asked of what, and how sure the answer was.
+ */
+export function templateReason ({ score, scope = null, about = null, verdict = null }) {
+  if (score == null) return null
+  const p = Math.round(score * 100)
+  const what = about === 'console' ? 'the console errors' : scope ? `the text of ${scope}` : 'the page text'
+  const sure = verdict === 'pass' || verdict === 'fail' ? 'clear enough for a verdict' : 'not sure either way'
+  return `Judged from ${what} without the screenshot: ${p} % yes, ${sure}.`
+}

package/src/lib/vision.mjs

@@ -0,0 +1,127 @@
+// Vision evaluator calls from the CLI (#17): screenshots scaled and cut in a browser page (WebP and PNG decode
+// there, no native dependency), one request per capture with the criteria that need it, cached by content like
+// the text evaluator. With ANTHROPIC_API_KEY (or the configured env) the call goes to the provider directly;
+// without it, through the organization's service, which counts it against the monthly allowance.
+import { join } from 'node:path'
+import { readFile } from 'node:fs/promises'
+import { readJson, writeJson, sha256 } from './util.mjs'
+import { buildVisionRequest, parseVisionResponse, visionHeaders, VISION_PROMPT_VERSION, MAX_VISION_IMAGES } from './vision-core.mjs'
+
+/**
+ * Images for one capture: tiles of the full page from the top (when a question needs the whole page) and a crop per
+ * scoped criterion (its highlight box, padded). Runs in `page` (a blank browser page).
+ * crops: [{ id, box: { x, y, w, h } }] in CSS px of the captured page; cssWidth: the page width in CSS px.
+ * @returns {Promise<{ images: Array<{ label, media_type, data, id? }>, failed: string[] }>} failed: crops that could not be cut (their
+ *   questions look at the full page)
+ */
+export async function prepareImages ({ page, file, cssWidth, crops = [], full = true, settings }) {
+  const src = `data:image/${file.endsWith('.png') ? 'png' : 'webp'};base64,${(await readFile(file)).toString('base64')}`
+  return page.evaluate(async ({ src, cssWidth, crops, full, maxW, tileH, maxTiles, maxImages }) => {
+    const img = await new Promise((ok, no) => { const i = new Image(); i.onload = () => ok(i); i.onerror = () => no(new Error('the screenshot could not be decoded')); i.src = src })
+    const k = img.naturalWidth / (cssWidth || img.naturalWidth)
+    const out = []
+    const encode = (c) => c.toDataURL('image/webp', 0.82).split(',')[1]
+    const draw = (sx, sy, sw, sh, dw, dh) => { const c = document.createElement('canvas'); c.width = Math.max(1, Math.round(dw)); c.height = Math.max(1, Math.round(dh)); const g = c.getContext('2d'); g.fillStyle = '#fff'; g.fillRect(0, 0, c.width, c.height); g.drawImage(img, sx, sy, sw, sh, 0, 0, c.width, c.height); return encode(c) }
+    const failed = []
+    for (const { id, box } of crops) {
+      if (out.length >= maxImages - (full ? maxTiles : 0)) { failed.push(id); continue }
+      const pad = 24
+      const sx = Math.max(0, (box.x - pad) * k); const sy = Math.max(0, (box.y - pad) * k)
+      const sw = Math.min(img.naturalWidth - sx, (box.w + pad * 2) * k); let sh = Math.min(img.naturalHeight - sy, (box.h + pad * 2) * k)
+      if (sw < 4 || sh < 4) { failed.push(id); continue }
+      const s = Math.min(1, maxW / sw)
+      if (sh * s > tileH * 1.2) sh = (tileH * 1.2) / s // a very tall region: its top part
+      out.push({ id, label: `crop ${id}`, media_type: 'image/webp', data: draw(sx, sy, sw, sh, sw * s, sh * s) })
+    }
+    if (full || failed.length) {
+      const s = Math.min(1, maxW / img.naturalWidth)
+      const H = img.naturalHeight * s
+      const parts = Math.ceil(H / tileH)
+      const tiles = Math.min(maxTiles, parts, maxImages - out.length)
+      const list = []
+      for (let t = 0; t < tiles; t++) {
+        const h = Math.min(tileH, H - t * tileH)
+        const label = parts === 1 ? 'page' : `page, part ${t + 1} of ${parts} from the top${t === tiles - 1 && tiles < parts ? ' (the rest of the page is not shown)' : ''}`
+        list.push({ label, media_type: 'image/webp', data: draw(0, (t * tileH) / s, img.naturalWidth, h / s, img.naturalWidth * s, h) })
+      }
+      out.unshift(...list)
+    }
+    return { images: out, failed }
+  }, { src, cssWidth, crops, full, maxW: settings.max_width, tileH: settings.tile_height, maxTiles: settings.max_tiles, maxImages: MAX_VISION_IMAGES })
+}
+
+/**
+ * The crops and questions for one capture: a scoped criterion (or one with a highlight) gets the crop of its first
+ * box unless that box is most of the page; the rest look at the full page.
+ */
+export function visionPlan ({ criteria, capture }) {
+  const page = capture.size ? capture.size.width * capture.size.height : 0
+  const crops = []; const questions = []
+  for (const c of criteria) {
+    const box = (c.check.scope || c.highlight) ? capture.boxes?.[c.id]?.[0] : null
+    const useCrop = box && box.w > 0 && box.h > 0 && (!page || (box.w * box.h) / page < 0.6)
+    if (useCrop) crops.push({ id: c.id, box })
+    questions.push({ id: c.id, question: c.check.question, ...(c.check.true ? { true: c.check.true } : {}), ...(c.check.false ? { false: c.check.false } : {}), ...(useCrop ? { image: `crop ${c.id}` } : {}) })
+  }
+  return { crops, questions, full: questions.some((q) => !q.image) }
+}
+
+/**
+ * Asks the vision evaluator with a content-addressed cache. Returns { answers, model, usage, cached, via } or
+ * { skipped } / { error }. `hosted(payload)` sends through the service when no local key is set; `allowCall()` is asked
+ * before every uncached request (false: skipped).
+ */
+export async function askVision ({ payload, settings, cacheDir, fetchImpl = fetch, hosted = null, allowCall = () => true }) {
+  const body = buildVisionRequest({ model: settings.model, effort: settings.effort, context: payload.context, questions: payload.questions, images: payload.images, reasonLanguage: settings.reason_language })
+  const key = sha256(JSON.stringify(body) + VISION_PROMPT_VERSION)
+  const cacheFile = join(cacheDir, `${key}.json`)
+  const cached = await readJson(cacheFile, null)
+  if (cached) return { ...cached, cached: true, key }
+  const apiKey = process.env[settings.api_key_env]
+  // uncached requests are bounded per evaluation (max_calls); the rest keep the text answer
+  if ((apiKey || (hosted && settings.hosted !== false)) && !allowCall()) return { skipped: `vision call limit reached (max_calls ${settings.max_calls})` }
+  if (!apiKey && hosted && settings.hosted !== false) {
+    const r = await hosted({ model: settings.model, effort: settings.effort, reason_language: settings.reason_language, ...payload })
+    if (r.skipped || r.error) return r
+    const out = { answers: r.response.answers || {}, model: r.response.model || settings.model, usage: r.response.usage || null, via: 'qawalk', hosted: r.response.qawalk || null }
+    await writeJson(cacheFile, out)
+    return { ...out, cached: !!r.response.qawalk?.cached, key }
+  }
+  if (!apiKey) return { skipped: `${settings.api_key_env} is not set${hosted ? '' : ' and the service is not configured (KALOKO_TOKEN)'}` }
+  const ids = payload.questions.map((q) => q.id)
+  for (let attempt = 0; attempt < 4; attempt++) {
+    let res
+    try {
+      res = await fetchImpl(`${settings.base_url.replace(/\/$/, '')}/v1/messages`, { method: 'POST', headers: visionHeaders(apiKey, body), body: JSON.stringify(body) })
+    } catch (error) { return { error: `vision evaluator unreachable: ${error.message}` } }
+    if (res.status === 429 || res.status === 529 || res.status === 503) {
+      const wait = Number(res.headers.get('retry-after')) * 1000 || 1000 * 2 ** attempt
+      await new Promise((r) => setTimeout(r, Math.min(wait, 20000)))
+      continue
+    }
+    if (!res.ok) return { error: `vision evaluator HTTP ${res.status}: ${(await res.text()).slice(0, 300)}` }
+    let parsed
+    try { parsed = parseVisionResponse(await res.json(), ids) } catch (error) { return { error: error.message } }
+    const out = { ...parsed, model: parsed.model || settings.model, via: 'own-key' }
+    await writeJson(cacheFile, out)
+    return { ...out, cached: false, key }
+  }
+  return { error: 'vision evaluator rate limited after 4 attempts' }
+}
+
+/**
+ * Sends a vision payload through the organization's service. An allowance used up or a service without hosted
+ * vision stops further hosted vision calls for this evaluation with one message.
+ */
+export function hostedVisionSender (config, report) {
+  let client = null
+  return async (payload) => {
+    if (report.visionStop) return { skipped: report.visionStop }
+    try { if (!client) { const { service } = await import('./service.mjs'); client = service(config) } } catch { return { skipped: 'no vision key (ANTHROPIC_API_KEY) and no service token (KALOKO_TOKEN) for hosted evaluation' } }
+    try { return { response: await client.request('POST', `/orgs/${config.org}/semantic/vision`, payload) } } catch (e) {
+      const m = String(e.message || e)
+      if (/HTTP (402|403|404|503)/.test(m)) { report.visionStop = m.replace(/^.*HTTP \d+: /, ''); return { skipped: report.visionStop } }
+      return { error: m }
+    }
+  }
+}

package/templates/kaloko.config.yml

@@ -69,6 +69,12 @@ evaluators:
     model: jev-1.13.0
     api_key_env: TYPESAFE_API_KEY
     thresholds: { pass: 0.85, fail: 0.15 }
+  # second look with the screenshot for criteria with vision: true and unsure text answers (false: off)
+  vision:
+    model: claude-opus-5-5
+    api_key_env: ANTHROPIC_API_KEY
+    when: uncertain
+    max_calls: 40
 
 share:
   url: https://kaloko.app