kaloko 0.4.8 → 0.4.10

package/src/lib/doctor.mjs

@@ -9,6 +9,7 @@ import { gateOf, gateHeaders, accountFor } from './access.mjs'
 import { missing } from './secrets.mjs'
 import { portFor } from './browser.mjs'
 import { DEFAULTS as JEV } from './jev.mjs'
+import { visionSettings, VISION_WEIGHT } from './vision-core.mjs'
 import { serviceUrl } from './service.mjs'
 
 const portFree = (port) => new Promise((done) => {
@@ -136,6 +137,12 @@ export async function doctor ({ config, root, file = null, fetchImpl = fetch, la
       else add('evaluator', false, s ? 'hosted evaluation not included in the plan' : 'hosted evaluation unavailable', `Put ${keyEnv} into .env or ~/.config/kaloko/.env to evaluate with your own key${s ? ', or upgrade the plan' : ''}.`)
     } else add('evaluator', false, `${keyEnv} not set`, `Semantic criteria will be "not evaluated" until ${keyEnv} is in .env or ~/.config/kaloko/.env${jev?.hosted === false ? '' : `, or ${tokenEnv} lets the service evaluate`}.`)
   }
+  // the vision evaluator is optional: without it unsure answers simply stay with a person
+  const vision = visionSettings(config)
+  if (!vision) add('vision evaluator', true, 'off in config')
+  else if (env[vision.api_key_env]) add('vision evaluator', true, `${vision.api_key_env} set · your key · model ${vision.model} · ${vision.when}`)
+  else if (token && vision.hosted !== false) add('vision evaluator', true, `via Kaloko · model ${vision.model} · one vision question counts as ${VISION_WEIGHT}`)
+  else add('vision evaluator', true, `${vision.api_key_env} not set (optional)`, `Criteria with vision: true and unsure answers stay with a person until ${vision.api_key_env} is set or ${tokenEnv} lets the service ask.`)
 
   return { ok: checks.every((c) => c.ok), checks }
 }

package/src/lib/drift.mjs

@@ -46,18 +46,24 @@ export function tokenOffenders (styles, tokens, mode = null, allowed = null) {
   return out.sort((a, b) => b.count - a.count)
 }
 
-/** Pixel difference ratio of two images in a browser page (the viewer's algorithm, so numbers match the canvas). */
-export async function pixelRatio (page, a, b, width) {
-  return page.evaluate(async ({ a, b, W }) => {
+/**
+ * Pixel difference ratio of two images in a browser page (the viewer's algorithm, so numbers match the canvas).
+ * ignore: { rects: [{x,y,w,h}] in CSS px, cssWidth } — regions left out of the count (#20, scenario `ignore`).
+ */
+export async function pixelRatio (page, a, b, width, ignore = null) {
+  return page.evaluate(async ({ a, b, W, rects, cssWidth }) => {
     const load = (src) => new Promise((ok, no) => { const i = new Image(); i.onload = () => ok(i); i.onerror = no; i.src = src })
     const [x, y] = await Promise.all([load(a), load(b)])
     const ha = Math.round(x.naturalHeight * W / x.naturalWidth), hb = Math.round(y.naturalHeight * W / y.naturalWidth), H = Math.max(ha, hb)
     const draw = (img, h) => { const c = document.createElement('canvas'); c.width = W; c.height = H; const g = c.getContext('2d', { willReadFrequently: true }); g.fillStyle = '#fff'; g.fillRect(0, 0, W, H); g.drawImage(img, 0, 0, W, h); return g.getImageData(0, 0, W, H).data }
     const da = draw(x, ha), db = draw(y, hb)
-    let n = 0
-    for (let i = 0; i < da.length; i += 4) if (Math.abs(da[i] - db[i]) + Math.abs(da[i + 1] - db[i + 1]) + Math.abs(da[i + 2] - db[i + 2]) > 48) n++
-    return { ratio: n / (W * H), heights: [ha, hb] }
-  }, { a, b, W: width })
+    const k = W / (cssWidth || W)
+    const skip = new Uint8Array(W * H)
+    for (const r of rects) { const x0 = Math.max(0, Math.floor(r.x * k)); const x1 = Math.min(W, Math.ceil((r.x + r.w) * k)); for (let yy = Math.max(0, Math.floor(r.y * k)); yy < Math.min(H, Math.ceil((r.y + r.h) * k)); yy++) skip.fill(1, yy * W + x0, yy * W + x1) }
+    let n = 0; let counted = 0
+    for (let i = 0, p = 0; i < da.length; i += 4, p++) { if (skip[p]) continue; counted++; if (Math.abs(da[i] - db[i]) + Math.abs(da[i + 1] - db[i + 1]) + Math.abs(da[i + 2] - db[i + 2]) > 48) n++ }
+    return { ratio: n / Math.max(1, counted), heights: [ha, hb], ignored: W * H - counted }
+  }, { a, b, W: width, rects: ignore?.rects || [], cssWidth: ignore?.cssWidth || null })
 }
 
 /**
@@ -96,7 +102,9 @@ export async function driftForCapture ({ node, entry, capture, runDir, ctx }) {
       const refPath = await cachedImage(ctx, rc.image)
       const mine = join(runDir, capture.files.image || capture.files.png)
       const toData = async (p) => `data:image/${p.endsWith('.png') ? 'png' : 'webp'};base64,${(await readFile(p)).toString('base64')}`
-      const { ratio } = await pixelRatio(ctx.page, await toData(mine), await toData(refPath), Math.min(1000, capture.size?.viewport?.[0] || 1280))
+      // regions the scenario ignores (clocks, ads) count in neither image
+      const ignore = { rects: [...(capture.ignore || []), ...(rc.ignore || [])], cssWidth: capture.size?.width || capture.size?.viewport?.[0] || null }
+      const { ratio } = await pixelRatio(ctx.page, await toData(mine), await toData(refPath), Math.min(1000, capture.size?.viewport?.[0] || 1280), ignore)
       const max = c.check.ratio ?? 0.03
       results[c.id] = { ok: ratio <= max, evidence: `${(ratio * 100).toFixed(1)} % of pixels differ from design v${ctx.referenceVersion ?? '?'} (limit ${(max * 100).toFixed(1)} %)`, ratio }
     }

package/src/lib/evaluate.mjs

@@ -1,7 +1,10 @@
 import { join, resolve } from 'node:path'
 import { readJson, writeJson, sha256 } from './util.mjs'
 import { readCaptures, runPaths } from './run.mjs'
-import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES } from './jev.mjs'
+import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
+import { visionSettings, visionCriteria, templateReason, VISION_PROMPT_VERSION } from './vision-core.mjs'
+import { prepareImages, visionPlan, askVision, hostedVisionSender } from './vision.mjs'
+import { KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
 import { packHelp } from './pack-help.mjs'
 
@@ -17,22 +20,37 @@ export const criteriaHash = (node) => sha256(node.criteria || [])
 export const stepIdOf = (e) => (e.item ? `${e.node}/items/${e.item}` : e.node)
 export const evaluationSource = (node, capture) => sha256({ capture, criteria: node.criteria || [], purpose: node.purpose || '' })
 
-/** Criterion verdicts for one capture, combining deterministic results and JEV answers. */
-export function combine ({ node, capture, jev, thresholds }) {
+/**
+ * Criterion verdicts for one capture, combining deterministic results, the text evaluator (JEV) and, where it was
+ * asked, the vision evaluator. Every semantic verdict with a score carries a one-sentence `reason`: the vision
+ * model's own, or a sentence saying what the text evaluator read (it returns a probability only).
+ */
+export function combine ({ node, capture, jev, vision = null, thresholds }) {
   return (node.criteria || []).map((c) => {
     if (c.check.type === 'deterministic') {
       const r = capture.checks?.[c.id]
       const verdict = r == null || r.ok == null ? 'not_evaluated' : r.ok ? 'pass' : 'fail'
-      const live = c.check.assert === 'no_console_errors' || c.check.assert === 'no_overflow' || c.check.assert === 'http_status' || c.check.assert === 'links_ok' || c.check.assert === 'hreflang_pairs' || /^perf_/.test(c.check.assert || '')
+      const live = c.check.assert === 'no_console_errors' || c.check.assert === 'no_overflow' || c.check.assert === 'http_status' || c.check.assert === 'links_ok' || c.check.assert === 'hreflang_pairs' || /^perf_/.test(c.check.assert || '') || KEYBOARD_ASSERTS.has(c.check.assert)
       return { id: c.id, type: 'deterministic', verdict, evidence: r?.evidence ?? null, ...(capture.rechecked_at && !live ? { rechecked: true } : {}) }
     }
     if (c.check.type === 'semantic' && c.check.about === 'console' && !(capture.console_errors || []).length && capture.checks) {
-      return { id: c.id, type: 'semantic', verdict: 'pass', evidence: 'no console errors', score: null, provider: null, note: null }
+      return { id: c.id, type: 'semantic', verdict: 'pass', evidence: 'no console errors', score: null, provider: null, note: null, reason: 'No console errors on this page, so there is nothing to triage.', reason_source: 'rule' }
     }
     if (c.check.type === 'semantic') {
       const answer = jev?.response?.answers?.[c.id]
-      const score = answer?.noul ?? null
-      return { id: c.id, type: 'semantic', verdict: verdictFor(score, thresholds), evidence: null, score, provider: answer ? 'typesafe-jev' : null, note: jev?.skipped || jev?.error || null }
+      const text = answer?.noul ?? null
+      const seen = vision?.answers?.[c.id] || null
+      const score = seen ? seen.probability : text
+      const verdict = verdictFor(score, thresholds)
+      const reason = seen?.reason || templateReason({ score, scope: c.check.scope, about: c.check.about, verdict })
+      return {
+        id: c.id, type: 'semantic', verdict, evidence: null, score,
+        provider: seen ? `${vision.provider || 'anthropic'}-vision` : answer ? 'typesafe-jev' : null,
+        note: score == null ? (jev?.skipped || jev?.error || vision?.skipped || vision?.error || null) : null,
+        ...(reason ? { reason, reason_source: seen?.reason ? 'model' : 'template' } : {}),
+        ...(seen && text != null ? { text_score: text } : {}),
+        ...(!seen && (vision?.skipped || vision?.error) && vision.ids?.includes(c.id) ? { vision_note: vision.skipped || vision.error } : {})
+      }
     }
     return { id: c.id, type: 'manual', verdict: 'not_evaluated', evidence: c.check.note || null }
   })
@@ -46,6 +64,12 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   const nodes = new Map(scenario.nodes.map((n) => [n.id, n]))
   const paths = runPaths(run.dir)
   const report = { evaluated: 0, semantic_calls: 0, semantic_cached: 0, skipped: [], model: settings.model, drift: 0, outdated: new Set(), via: {}, hosted: null, hostedStop: null, findings: [] }
+  // the vision evaluator (#17): flagged and uncertain criteria, a bounded number of calls per evaluation
+  const vset = visionSettings(config)
+  if (vset) report.vision = { model: vset.model, asked: 0, calls: 0, cached: 0, skipped: [], via: {}, hosted: null }
+  // with an own text-evaluator key the vision evaluator uses an own key too, unless hosted: true says otherwise
+  if (vset && process.env[settings.api_key_env] && config.evaluators?.vision?.hosted !== true) vset.hosted = false
+  const vision = vset ? visionContext({ config, root, settings: vset, report, fetchImpl }) : null
   // checks against the design reference and the token set (pack design, #42) need the service and a browser
   const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
   try {
@@ -77,8 +101,10 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
       const semantic = outOfSample ? [] : (node.criteria || []).filter((c) => c.check.type === 'semantic' && !(c.check.about === 'console' && !(capture.console_errors || []).length))
       // quick signals ride along on every captured page state (also steps without semantic criteria)
       const signals = settings.signals !== false && !outOfSample && !!capture.state
-      const request = (semantic.length || signals) && current ? buildRequest({ node, capture, criteria: semantic, model: settings.model, signals }) : null
-      work.push({ entry, node, capture, current, outOfSample, request, asks: semantic.length > 0, jev: current ? null : { skipped: 'Capture has no matching criteria fingerprint; capture this step again.' } })
+      // an imported screen without page text (#11) has nothing to ask about: a person decides on the canvas
+      const textless = capture.capture?.source === 'import' && !capture.state
+      const request = (semantic.length || signals) && current && !textless ? buildRequest({ node, capture, criteria: semantic, model: settings.model, signals }) : null
+      work.push({ entry, node, capture, current, outOfSample, request, asks: semantic.length > 0, jev: !current ? { skipped: 'Capture has no matching criteria fingerprint; capture this step again.' } : textless && semantic.length ? { skipped: 'imported screenshot without page text; a person decides on the canvas' } : null })
       if (!current) report.outdated.add(entry.node)
     }
 
@@ -91,10 +117,12 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
       }
     }))
-    for (const { entry, node, capture, current, outOfSample, request, jev } of work) {
+    // second opinion with the screenshot for flagged and uncertain criteria
+    if (vision) await vision.ask(work.filter((w) => w.current && !w.outOfSample && w.request), run.dir)
+    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen } of work) {
       const source = evaluationSource(node, capture)
       if (drift && current && needsDrift(node)) { capture.checks = { ...capture.checks, ...await driftForCapture({ node, entry, capture, runDir: run.dir, ctx: drift }) }; report.drift++ }
-      let criteria = combine({ node, capture: current ? capture : {}, jev, thresholds: settings.thresholds })
+      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, thresholds: settings.thresholds })
       // pages outside the semantic sample carry only the deterministic results
       if (outOfSample) criteria = criteria.filter((c) => c.type !== 'semantic' || c.evidence === 'no console errors')
       const evaluation = {
@@ -115,16 +143,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         thresholds: settings.thresholds,
         state_chars: request ? JSON.stringify(request.state).length : null,
         ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
+        ...(seen ? { vision: seen.answers ? { provider: seen.provider, via: seen.via, model: seen.model, prompt_version: VISION_PROMPT_VERSION, cached: seen.cached, usage: seen.usage || null, answers: seen.answers } : { provider: seen.provider, skipped: seen.skipped || null, error: seen.error || null } } : {}),
         criteria
       }
       await writeJson(paths.step(stepIdOf(entry), entry.locale, entry.viewport, 'eval.json'), evaluation)
       report.evaluated++
       // what did not pass, for the walk summary and agents: step, locale, viewport, criterion and why
-      for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes` : c.note) || null, ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
+      for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes${c.reason_source === 'model' ? `: ${c.reason}` : ''}` : c.note) || null, ...(c.reason ? { reason: c.reason } : {}), ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
     }
   }
-  } finally { if (drift?.browser) await drift.browser.close() }
-  const { hostedStop, ...out } = report
+  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close() }
+  const { hostedStop, visionStop, ...out } = report
   return { ...out, outdated: [...report.outdated] }
 }
 
@@ -184,6 +213,61 @@ export function semanticPrefetch ({ scenario, config, root }) {
   }
 }
 
+/**
+ * The vision step of an evaluation: picks the criteria per capture, cuts the images in a lazily started browser
+ * and asks, a few captures at a time, within `max_calls` uncached requests. Fills `w.vision` on each work item.
+ */
+function visionContext ({ config, root, settings, report, fetchImpl }) {
+  const cacheDir = join(resolve(root, config.output), '.cache', 'vision')
+  const hosted = hostedVisionSender(config, report)
+  let browser = null; let pagePromise = null
+  const page = () => (pagePromise ||= (async () => {
+    const { launchEphemeral } = await import('./browser.mjs')
+    browser = await launchEphemeral(config)
+    const p = await browser.newPage()
+    await p.setContent('<!doctype html><title>vision</title>')
+    return p
+  })())
+  const allowCall = () => { if (report.vision.calls >= (settings.max_calls ?? 40)) return false; report.vision.calls++; return true }
+  // no key and no service token: nothing to ask, so no browser either
+  const unavailable = process.env[settings.api_key_env] ? null : settings.hosted === false ? `${settings.api_key_env} is not set` : !process.env[config.share?.token_env || 'KALOKO_TOKEN'] ? `${settings.api_key_env} is not set and the service is not configured (KALOKO_TOKEN)` : null
+  return {
+    async ask (work, runDir) {
+      const queue = []
+      for (const w of work) {
+        const semantic = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && w.request.questions[c.id])
+        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings })
+        if (crit.length) queue.push({ w, crit })
+      }
+      await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 3, queue.length)) }, async () => {
+        for (let job = queue.shift(); job; job = queue.shift()) {
+          const { w, crit } = job
+          const ids = crit.map((c) => c.id)
+          const file = w.capture.files?.image || w.capture.files?.png
+          report.vision.asked++
+          if (unavailable) { w.vision = { provider: settings.provider, skipped: unavailable, ids }; report.vision.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${unavailable}`); continue }
+          if (!file) { w.vision = { provider: settings.provider, skipped: 'no screenshot', ids }; continue }
+          if (report.visionStop && !process.env[settings.api_key_env]) { w.vision = { provider: settings.provider, skipped: report.visionStop, ids }; continue }
+          try {
+            const plan = visionPlan({ criteria: crit, capture: w.capture })
+            const { images, failed } = await prepareImages({ page: await page(), file: join(runDir, file), cssWidth: w.capture.size?.width, crops: plan.crops, full: plan.full, settings })
+            for (const q of plan.questions) if (failed.includes(q.id)) delete q.image
+            const payload = { context: { purpose: purposeOf(w.node, w.capture), url: w.capture.url, locale: w.capture.locale, viewport: w.entry.viewport }, questions: plan.questions, images }
+            const r = await askVision({ payload, settings, cacheDir, fetchImpl, hosted, allowCall })
+            w.vision = { provider: settings.provider, ids, ...r }
+            if (r.answers) { if (r.cached) report.vision.cached++; report.vision.via[r.via] = (report.vision.via[r.via] || 0) + 1; if (r.hosted) report.vision.hosted = r.hosted }
+            else report.vision.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${r.skipped || r.error}`)
+          } catch (error) {
+            w.vision = { provider: settings.provider, error: String(error.message || error).slice(0, 300), ids }
+            report.vision.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.vision.error}`)
+          }
+        }
+      }))
+    },
+    async close () { if (browser) await browser.close().catch(() => {}) }
+  }
+}
+
 /**
  * Sends a JEV request through the organization's service (hosted evaluation) when a service token is set.
  * An allowance used up or a service without hosting stops further hosted calls for this run with one message.

package/src/lib/git.mjs

@@ -59,12 +59,16 @@ export async function gitMeta (cwd) {
     out('git', ['rev-parse', 'HEAD'], cwd)
   ])
   let pr = null
-  const json = await out('gh', ['pr', 'view', '--json', 'number,url,state'], cwd)
+  const vars = process.env
+  // CI checks out a detached commit: the branch and pull request come from the environment
+  const ciBranch = vars.KALOKO_BRANCH || vars.GITHUB_HEAD_REF || vars.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || vars.CI_COMMIT_REF_NAME || null
+  if (vars.GITLAB_CI && vars.CI_MERGE_REQUEST_IID) pr = { number: Number(vars.CI_MERGE_REQUEST_IID), url: `${vars.CI_MERGE_REQUEST_PROJECT_URL || vars.CI_PROJECT_URL}/-/merge_requests/${vars.CI_MERGE_REQUEST_IID}`, state: 'open' }
+  const json = pr ? null : await out('gh', ['pr', 'view', ...(vars.KALOKO_PR ? [String(vars.KALOKO_PR)] : []), '--json', 'number,url,state'], cwd)
   if (json) {
     try {
       const p = JSON.parse(json)
       pr = { number: p.number, url: p.url, state: String(p.state).toLowerCase() }
     } catch {}
   }
-  return { branch, commit, sha, message, author, pr }
+  return { branch: branch === 'HEAD' && ciBranch ? ciBranch : branch, commit, sha, message, author, pr }
 }

package/src/lib/heal.mjs

@@ -0,0 +1,207 @@
+// Self-healing selectors (#13): when a step script waits in vain for a selector, propose a replacement from the page
+// as it is now (role and accessible name, test ids, text), preferring the element the old selector matched in the
+// step's previous capture. The proposal is printed as a patch; `walk --heal` applies it to the script only after the
+// step passed with it. Nothing is ever changed silently.
+import { readFile, writeFile, rm, readdir } from 'node:fs/promises'
+import { join, dirname, basename, relative, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { readJson } from './util.mjs'
+
+/** The selector a Playwright timeout waited for, or null (only plain locator strings can be patched). */
+export function failedSelector (message) {
+  const text = String(message || '')
+  if (!/Timeout \d+ms exceeded|waiting for/i.test(text) || /strict mode violation/i.test(text)) return null
+  const unq = (s, q) => s.replace(new RegExp(`\\\\${q}`, 'g'), q)
+  let m = text.match(/waiting for locator\('((?:\\'|[^'])+)'\)/)
+  if (m) return unq(m[1], "'")
+  m = text.match(/waiting for locator\("((?:\\"|[^"])+)"\)/)
+  if (m) return unq(m[1], '"')
+  m = text.match(/waiting for selector "((?:\\"|[^"])+)"/)
+  if (m) return unq(m[1], '"')
+  return null
+}
+
+/** Words a selector is made of: ids, classes, attribute values and text, split on punctuation and camelCase. */
+export function tokensOf (selector) {
+  return [...new Set(String(selector)
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[^a-z0-9áčďéěíňóřšťúůýž]+/i)
+    .filter((w) => w.length > 1 && !['text', 'has', 'nth', 'first', 'last', 'visible', 'true', 'role', 'name', 'internal', 'data', 'testid', 'test', 'id', 'qa', 'cy', 'div', 'span'].includes(w)))]
+}
+
+/**
+ * In the page (self-contained, passed to evaluate): interactive and labelled elements with role, accessible name,
+ * test id, id and classes; given an element, that element only.
+ */
+function describeAll (limit) {
+  const TEST = ['data-testid', 'data-test', 'data-qa', 'data-cy', 'data-test-id']
+  const roleOf = (el) => {
+    const r = el.getAttribute('role'); if (r) return r
+    const t = el.tagName.toLowerCase()
+    if (t === 'a' && el.hasAttribute('href')) return 'link'
+    if (t === 'button' || (t === 'input' && ['button', 'submit', 'reset'].includes(el.type))) return 'button'
+    if (t === 'input') return { checkbox: 'checkbox', radio: 'radio', range: 'slider', search: 'searchbox' }[el.type] || 'textbox'
+    if (t === 'textarea') return 'textbox'
+    if (t === 'select') return 'combobox'
+    if (/^h[1-6]$/.test(t)) return 'heading'
+    if (t === 'img') return 'img'
+    return null
+  }
+  const text = (s) => String(s || '').replace(/\s+/g, ' ').trim()
+  const nameOf = (el) => {
+    const l = el.getAttribute('aria-label'); if (l) return text(l)
+    const by = el.getAttribute('aria-labelledby'); if (by) { const t = by.split(/\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' '); if (text(t)) return text(t) }
+    if (el.id) { const lab = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); if (lab) return text(lab.textContent) }
+    const wrap = el.closest('label'); if (wrap && wrap !== el && /^(input|select|textarea)$/i.test(el.tagName)) return text(wrap.textContent)
+    if (/^(input|textarea)$/i.test(el.tagName)) return text(el.getAttribute('placeholder') || el.getAttribute('title') || (['button', 'submit'].includes(el.type) ? el.value : ''))
+    if (el.tagName === 'IMG') return text(el.getAttribute('alt'))
+    return text(el.innerText || el.textContent).slice(0, 80)
+  }
+  const describe = (el) => {
+    const testid = TEST.map((a) => el.getAttribute(a) ? [a, el.getAttribute(a)] : null).find(Boolean) || null
+    const r = el.getBoundingClientRect(); const cs = getComputedStyle(el)
+    return { tag: el.tagName.toLowerCase(), role: roleOf(el), name: nameOf(el), testid, id: el.id || null, classes: [...el.classList].slice(0, 8), visible: r.width > 0 && r.height > 0 && cs.visibility !== 'hidden' && cs.display !== 'none' }
+  }
+  if (typeof limit === 'object' && limit?.nodeType === 1) return describe(limit)
+  const sel = 'a[href], button, input, select, textarea, summary, [role], [aria-label], [data-testid], [data-test], [data-qa], [data-cy], [data-test-id], h1, h2, h3, label'
+  return [...document.querySelectorAll(sel)].slice(0, limit).map(describe).filter((d) => d.visible && (d.name || d.testid || d.id))
+}
+
+/** The element the old selector matched in the previous capture of this step (its HTML), described; null if none. */
+async function previousFingerprint ({ page, selector, root, output, run, node, locale }) {
+  const base = resolve(root, output)
+  let names = []
+  try { names = (await readdir(base, { withFileTypes: true })).filter((e) => e.isDirectory() && e.name !== run.meta.id && !e.name.startsWith('.')).map((e) => e.name).sort().reverse() } catch { return null }
+  for (const name of names.slice(0, 20)) {
+    const meta = await readJson(join(base, name, 'meta.json'), null)
+    if (meta?.scenario?.id !== run.meta.scenario.id) continue
+    let html = null
+    for (const vp of Object.keys(run.meta.scope.viewports || {})) { html = await readFile(join(base, name, 'steps', node.id, locale, `${vp}.html`), 'utf8').catch(() => null); if (html) break }
+    if (!html) continue
+    let scratch = null
+    try {
+      const browser = page.context().browser()
+      scratch = browser ? await browser.newContext({ javaScriptEnabled: false }) : null
+      if (!scratch) return null
+      const p = await scratch.newPage()
+      await p.route('**/*', (r) => r.abort())
+      await p.setContent(html.replace(/<script[\s\S]*?<\/script>/gi, ''), { waitUntil: 'domcontentloaded', timeout: 5000 })
+      const el = p.locator(selector).first()
+      if (!(await el.count().catch(() => 0))) continue
+      const fp = await el.evaluate(describeAll)
+      return { ...fp, run: name }
+    } catch { continue } finally { await scratch?.close().catch(() => {}) }
+  }
+  return null
+}
+
+const q = (s) => JSON.stringify(String(s))
+
+/** Scores a candidate against the fingerprint of the old element and the words of the old selector. */
+export function score (c, { fp, tokens }) {
+  let s = 0; const why = []
+  if (fp) {
+    if (fp.testid && c.testid && fp.testid[1] === c.testid[1]) { s += 6; why.push(`same ${c.testid[0]} "${c.testid[1]}"`) }
+    if (fp.name && c.name && fp.name.toLowerCase() === c.name.toLowerCase()) { s += 4; why.push(`same name "${c.name}"`) }
+    if (fp.role && fp.role === c.role) { s += 2; why.push(`role ${c.role}`) }
+    if (fp.tag === c.tag) s += 1
+    if (fp.id && fp.id === c.id) { s += 3; why.push(`same id ${c.id}`) }
+    const shared = (fp.classes || []).filter((k) => c.classes.includes(k)).length
+    s += Math.min(2, shared)
+  }
+  const hay = [c.id, ...(c.classes || []), c.testid?.[1], c.name, c.role, c.tag].filter(Boolean).join(' ').toLowerCase()
+  const words = tokens.filter((t) => hay.includes(t))
+  if (words.length) { s += words.length * 1.5; why.push(`words ${words.map((w) => `"${w}"`).join(', ')} of the old selector`) }
+  return { score: s, why }
+}
+
+/** Selectors for an element, most stable first: test id, role + name, id, text. */
+export function selectorsFor (c) {
+  const out = []
+  if (c.testid) out.push(`[${c.testid[0]}=${q(c.testid[1])}]`)
+  if (c.role && c.name) out.push(`role=${c.role}[name=${q(c.name)}]`)
+  if (c.id && !/\d{3,}|^[a-f0-9-]{16,}$|^(radix|headlessui|mui|react-aria|:r)/i.test(c.id)) out.push(`#${c.id}`)
+  if (c.name && c.name.length <= 60) out.push(`text=${q(c.name)}`)
+  return out
+}
+
+/**
+ * A replacement for `selector` on the page as it is now: { replacement, why, from } or null.
+ * The replacement matches exactly one element.
+ */
+export async function proposeSelector ({ page, selector, root, output, run, node, locale }) {
+  const fp = await previousFingerprint({ page, selector, root, output, run, node, locale }).catch(() => null)
+  const tokens = tokensOf(selector)
+  let candidates
+  try { candidates = await page.evaluate(describeAll, 1500) } catch { return null }
+  const ranked = candidates.map((c) => ({ c, ...score(c, { fp, tokens }) })).filter((x) => x.score >= (fp ? 4 : 3)).sort((a, b) => b.score - a.score)
+  for (const { c, why } of ranked.slice(0, 8)) {
+    for (const sel of selectorsFor(c)) {
+      if (sel === selector) continue
+      const n = await page.locator(sel).count().catch(() => 0)
+      if (n === 1) return { replacement: sel, why: why.join(', ') || 'closest element', from: fp?.run || null, element: { role: c.role, name: c.name, tag: c.tag } }
+    }
+  }
+  return null
+}
+
+/** Replaces the selector's string literal in a script; { text, diff, count } or null when it is not written literally. */
+export function patchScript (source, selector, replacement, file = 'script.mjs') {
+  const lines = source.split('\n')
+  const out = []; const diff = []; let count = 0
+  for (const [i, line] of lines.entries()) {
+    let next = line
+    for (const quote of ["'", '"', '`']) {
+      const lit = quote + selector.replaceAll('\\', '\\\\').replaceAll(quote, '\\' + quote) + quote
+      if (next.includes(lit)) {
+        const rep = quote + replacement.replaceAll('\\', '\\\\').replaceAll(quote, '\\' + quote) + quote
+        count += next.split(lit).length - 1
+        next = next.replaceAll(lit, rep)
+      }
+    }
+    if (next !== line) diff.push(`@@ -${i + 1} +${i + 1} @@`, `-${line}`, `+${next}`)
+    out.push(next)
+  }
+  if (!count) return null
+  return { text: out.join('\n'), count, diff: [`--- a/${file}`, `+++ b/${file}`, ...diff].join('\n') }
+}
+
+/**
+ * After a failed step: the proposal and, with `apply`, a re-run of the step with the patched script. `rerun(module)`
+ * runs the step with a script module and throws when it fails. Returns { selector, proposal, patch, applied, note }.
+ */
+export async function healStep ({ error, page, scriptFile, root, output, run, node, locale, apply = false, rerun }) {
+  const selector = failedSelector(error?.message)
+  if (!selector) return null
+  const proposal = await proposeSelector({ page, selector, root, output, run, node, locale })
+  if (!proposal) return { selector, proposal: null, note: `no element on the page now resembles ${selector}` }
+  const source = await readFile(scriptFile, 'utf8').catch(() => null)
+  const rel = relative(root, scriptFile)
+  const patch = source ? patchScript(source, selector, proposal.replacement, rel) : null
+  if (!patch) return { selector, proposal, patch: null, note: `${selector} is not written literally in ${rel}; change it by hand` }
+  if (!apply) return { selector, proposal, patch, applied: false }
+  // the patched script runs from a sibling file (its relative imports resolve the same) and replaces the original
+  // only when the step passed with it
+  const trial = join(dirname(scriptFile), `.${basename(scriptFile, '.mjs')}.kaloko-heal-${process.pid}.mjs`)
+  try {
+    await writeFile(trial, patch.text)
+    await rerun(await import(`${pathToFileURL(trial).href}?t=${Date.now()}`))
+    await writeFile(scriptFile, patch.text)
+    return { selector, proposal, patch, applied: true }
+  } catch (e) {
+    return { selector, proposal, patch, applied: false, note: `the step still fails with the proposal (${String(e.message).split('\n')[0]}); ${rel} is unchanged` }
+  } finally { await rm(trial, { force: true }) }
+}
+
+/** Lines for the terminal: what failed, what is proposed and why, and the patch. */
+export function formatHeal (h, { stepArg }) {
+  if (!h) return ''
+  if (!h.proposal) return `  ↳ selector ${h.selector}: ${h.note}`
+  const lines = [`  ↳ selector ${h.selector} not found; proposed ${h.proposal.replacement} (${h.proposal.why}${h.proposal.from ? `; the old selector matched it in run ${h.proposal.from}` : ''})`]
+  if (h.applied) lines.push(`  ↳ healed: the step passed with it, ${h.patch.count} occurrence(s) changed:`)
+  else if (h.note) lines.push(`  ↳ ${h.note}`)
+  if (h.patch) lines.push(h.patch.diff.split('\n').map((l) => `    ${l}`).join('\n'))
+  if (h.patch && !h.applied && !h.note) lines.push(`  ↳ apply it after the step passes: kaloko walk --steps ${stepArg} --heal`)
+  return lines.join('\n')
+}