kaloko 0.11.0 → 0.12.1

package/src/lib/capture.mjs

@@ -5,6 +5,7 @@ import { probe } from './probe.mjs'
 import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
+import { maskSpec, maskList, inPage, maskInPage, rememberPii, knownPii, maskMailHeaders, maskHtml, scrubDeep, scrubText, updateMasks, loadKnownPii, saveKnownPii, RECORD_PLAIN_KEYS } from './pii.mjs'
 
 /**
  * Encodes a PNG screenshot inside the page (no native dependency): the image as WebP (or PNG, lossless) and a
@@ -16,7 +17,14 @@ export async function encodeImage (page, png, { format = 'webp', quality = 0.82,
     await new Promise((res, rej) => { img.onload = res; img.onerror = rej; img.src = dataUrl })
     const full = document.createElement('canvas'); full.width = img.width; full.height = img.height
     const g = full.getContext('2d'); g.drawImage(img, 0, 0)
-    g.fillStyle = '#1f1f1d'; for (const r of masks) g.fillRect(Math.floor(r.x * scale), Math.floor(r.y * scale), Math.ceil(r.w * scale), Math.ceil(r.h * scale))
+    // masked personal data: a dark box with diagonal stripes, so a reader sees "hidden" and not a broken page (#64)
+    for (const r of masks) {
+      const x = Math.floor(r.x * scale); const y = Math.floor(r.y * scale); const w = Math.ceil(r.w * scale); const h = Math.ceil(r.h * scale)
+      g.fillStyle = '#1f1f1d'; g.fillRect(x, y, w, h)
+      g.save(); g.beginPath(); g.rect(x, y, w, h); g.clip(); g.strokeStyle = '#3d3d38'; g.lineWidth = 3 * scale
+      for (let i = -h; i < w; i += 12 * scale) { g.beginPath(); g.moveTo(x + i, y + h); g.lineTo(x + i + h, y); g.stroke() }
+      g.restore()
+    }
     for (const m of marks) {
       const r = 13 * scale; const x = Math.max(r + 2, m.x * scale); const y = Math.max(r + 2, m.y * scale)
       g.beginPath(); g.arc(x, y, r, 0, Math.PI * 2); g.fillStyle = '#ffe452'; g.fill(); g.lineWidth = 2 * scale; g.strokeStyle = '#151514'; g.stroke()
@@ -50,12 +58,15 @@ export async function stepSpec (page, tokens = null, mode = null) {
 
 export function captureOptions (scenario, node) {
   const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
+  // masks add up: a step's own selectors never drop the scenario's (personal data stays hidden on every step)
+  const masks = [maskList(scenario?.capture?.mask), maskList(node?.capture?.mask)]
   const q = Number(o.quality)
   return {
     format: o.format === 'png' ? 'png' : 'webp',
     scale: Number(o.scale) === 2 ? 2 : 1,
     quality: Number.isFinite(q) && q > 0 ? Math.min(1, Math.max(0.3, q)) : 0.82,
-    mask: Array.isArray(o.mask) ? o.mask.map(String).filter(Boolean) : [],
+    mask: [...new Set(masks.flatMap((m) => m.selectors))],
+    maskAuto: masks.some((m) => m.auto),
     annotate: o.annotate === true,
     // stack pages default to the first screen: hundreds of full-page shots would swamp storage
     fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack'
@@ -121,6 +132,11 @@ export async function captureStep ({ page, node, locale, viewports, run, console
   const original = page.viewportSize()
   const results = []
   let netResults = null
+  // personal data (#64): what this capture masks; recipients of a captured e-mail are masked before the body
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  // values masked by an earlier process of this run (kaloko walk, then kaloko capture) are masked here too
+  if (mask.active) await loadKnownPii(run.dir)
+  if (mask.active && extra.mail) extra = { ...extra, mail: maskMailHeaders(extra.mail, mask) }
   // batch E: the engine of this page, and per key its colour scheme, motion and display mode (run scope variants)
   const engine = engineOf(page)
   const scope = run.meta?.scope
@@ -128,14 +144,58 @@ export async function captureStep ({ page, node, locale, viewports, run, console
     const now = page.viewportSize()
     if (!now || now.width !== width || now.height !== height) await page.setViewportSize({ width, height })
     const undoVariant = await applyVariant(page, scope?.variants?.[viewport], scope?.devices?.[scope?.variants?.[viewport]?.device] || null)
+    let masking = null
     try {
       await settle(page)
       const html = paths.step(stepId, locale, viewport, 'html')
       await rm(paths.step(stepId, locale, viewport, 'eval.json'), { force: true })
       await mkdir(dirname(html), { recursive: true })
-      // PII selectors are covered by a solid box; at 2× Chrome renders the full page at double scale (CDP), and
-      // the boxes are drawn while encoding because Playwright's own mask only works at the page's pixel ratio
-      const maskRects = opts.mask.length ? await page.evaluate((sels) => sels.flatMap((sel) => { try { return [...document.querySelectorAll(sel)] } catch { return [] } }).map((el) => { const r = el.getBoundingClientRect(); return { x: r.left + scrollX, y: r.top + scrollY, w: r.width, h: r.height } }).filter((r) => r.w > 0 && r.h > 0), opts.mask) : []
+      // the checks read the page as it is: masks change what is stored, never a verdict
+      let result
+      for (let attempt = 0; ; attempt++) {
+        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
+          // the page navigated under us (redirect, Turbo); wait once and probe the new document
+          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
+          throw error
+        }
+      }
+      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
+      const ignore = await resolveIgnore(page, node, locale, viewport)
+      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the mask and the keyboard walk
+      await liveValueChecks({ page, node, result, url: page.url(), tokens })
+      // the console is read before the mask: the page's own reaction to masked text never counts as its error
+      const url = page.url()
+      const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
+      const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
+      // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
+      // an error page's own status ("Failed to load resource: … 404" for the document itself) is not a console error
+      const ownStatus = (e) => result.http_status >= 400 && /^Failed to load resource: the server responded with a status of \d+/.test(e.text) && e.src && sameDocument(e.src, url)
+      const errors = consoleEntries.filter((e) => e.type !== 'blocked' && sameDocument(e.url, url) && !e.text.includes('ERR_BLOCKED_BY_CLIENT') && !ownStatus(e)).map((e) => e.text)
+      const blocked = consoleEntries.filter((e) => e.type === 'blocked').map((e) => `${e.text}`)
+      for (const c of node.criteria || []) {
+        if (c.check.type === 'deterministic' && c.check.assert === 'no_console_errors') {
+          result.checks[c.id] = collected === null
+            ? { ok: null, evidence: 'Console was not observed.' }
+            : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
+        }
+      }
+      // One mask for every output (#64): from here until the HTML view the page shows placeholders or fake values
+      // instead of personal data, so the screenshot, HTML, outline, spec and HTML view agree
+      if (mask.active) {
+        masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
+        rememberPii(masking.pairs)
+        await saveKnownPii(run.dir)
+        await updateMasks(page)
+        // the outline the evaluators read comes from the masked page; checks and boxes keep the first probe. When the
+        // masked page cannot be read, nothing is stored: the first probe holds the real data
+        if (masking.changed) {
+          let again = null
+          try { again = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)) } catch {}
+          if (!again) throw new Error(`capture: the masked page of ${node.id} could not be read again; nothing was stored so no personal data leaves in the page outline`)
+          for (const [k, v] of Object.entries(again)) if (k !== 'checks' && k !== 'boxes' && k !== 'http_status') result[k] = v
+        }
+      }
+      const maskRects = (masking?.rects || []).filter((r) => r.box)
       let shot
       // a device preset's pixel ratio (3 on an iPhone) shapes the layout, not the image: captures stay in CSS pixels
       // (or 2× for documentation), so the canvas, boxes and diffs work the same for every device
@@ -152,22 +212,12 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         // WebKit and Firefox have no 2× capture of their own: 2× when the context renders at 2× or more, else 1×
         // (exactly 2×: a 3× or 2.625× device would give an image the masks and marks are not drawn for)
         if (opts.scale === 2 && dpr !== 2) shotScale = 1
-        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css', ...(opts.mask.length && shotScale === 1 ? { mask: opts.mask.map((sel) => page.locator(sel)), maskColor: '#1f1f1d' } : {}) })
+        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css' })
       }
       // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
-      await writeFile(html, redact(await page.content()))
-      let result
-      for (let attempt = 0; ; attempt++) {
-        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
-          // the page navigated under us (redirect, Turbo); wait once and probe the new document
-          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
-          throw error
-        }
-      }
+      await writeFile(html, redact(maskHtml(await page.content(), masking?.attrs)))
       // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
       if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens, scope?.variants?.[viewport]?.token_mode || scope?.variants?.[viewport]?.color_scheme || null)
-      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
-      const ignore = await resolveIgnore(page, node, locale, viewport)
       // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
       const unstable = stability > 1 ? await repeatRegions(page, { n: Math.min(6, stability), fullPage: opts.fullPage }).catch((error) => ({ error: String(error.message).slice(0, 200) })) : null
       // numbered marks on the highlighted elements, in criteria order (first box of each)
@@ -176,7 +226,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       let thumb = paths.step(stepId, locale, viewport, 'thumb.webp')
       let thumbWritten = true
       try {
-        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: shotScale === 2 ? maskRects : [] })
+        // mask boxes are drawn while encoding, at the image's own scale (Playwright's mask only works at the page's pixel ratio)
+        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: maskRects })
         // the extension follows the bytes (WebKit encodes PNG), so the service accepts the file
         if (enc.imageType === 'image/png') image = paths.step(stepId, locale, viewport, 'png')
         if (enc.thumbType === 'image/png') thumb = paths.step(stepId, locale, viewport, 'thumb.png')
@@ -184,33 +235,31 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         await writeFile(thumb, enc.thumb)
       } catch (error) {
         // never store an unmasked image when masks were asked for
-        if (shotScale === 2 && maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
+        if (maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
         image = paths.step(stepId, locale, viewport, 'png'); thumbWritten = false
         await writeFile(image, shot)
       }
-      const url = page.url()
-      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the keyboard walk moves focus
-      await liveValueChecks({ page, node, result, url, tokens })
-      const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
-      const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
-      // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
-      // an error page's own status ("Failed to load resource: … 404" for the document itself) is not a console error
-      const ownStatus = (e) => result.http_status >= 400 && /^Failed to load resource: the server responded with a status of \d+/.test(e.text) && e.src && sameDocument(e.src, url)
-      const errors = consoleEntries.filter((e) => e.type !== 'blocked' && sameDocument(e.url, url) && !e.text.includes('ERR_BLOCKED_BY_CLIENT') && !ownStatus(e)).map((e) => e.text)
-      const blocked = consoleEntries.filter((e) => e.type === 'blocked').map((e) => `${e.text}`)
       // network checks (internal links, hreflang pairs) run once per page, outside the page, with a walk-wide cache
       const net = (node.criteria || []).filter((c) => c.check.type === 'deterministic' && NETWORK_ASSERTS.has(c.check.assert))
       if (net.length) {
         netResults ||= await networkChecks(page, net, url, { run, locale })
         Object.assign(result.checks, netResults)
       }
-      for (const c of node.criteria || []) {
-        if (c.check.type === 'deterministic' && c.check.assert === 'no_console_errors') {
-          result.checks[c.id] = collected === null
-            ? { ok: null, evidence: 'Console was not observed.' }
-            : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
-        }
+      // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
+      let view = null
+      if (htmlView) {
+        try {
+          // the page is still masked here (#64); the selectors are covered in the copy as well (#27)
+          const attrs = masking?.attrs
+          const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])], scrub: (text) => redact(maskHtml(text, attrs)) })
+          await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
+          view = { revision: rev.id, entry: rev.entry }
+        } catch (error) { view = { error: String(error.message).slice(0, 200) } }
       }
+      // the page gets its own text back before the keyboard walk: Tab and the accessibility tree see the real page,
+      // and only the focus shots are taken masked
+      const maskedPage = masking
+      if (masking) { await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}); masking = null }
       // keyboard and screen-reader checks (#19): a Tab walk and the accessibility tree, after the screenshot and after
       // the console was read (focus handlers of the page must not count as its errors)
       let keyboard = null
@@ -225,22 +274,16 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       }
       if (needsKeyboard(kbNode) && kbSampled) {
         const kb = await keyboardChecks(page, kbNode.criteria, {
-          saveShot: async (n, png) => { const f = paths.step(stepId, locale, viewport, `focus-${n}.png`); await writeFile(f, png); return relative(run.dir, f) }
+          saveShot: async (n, png, clip) => {
+            const f = paths.step(stepId, locale, viewport, `focus-${n}.png`)
+            await writeFile(f, mask.active ? await maskedShot(page, mask, clip) : png)
+            return relative(run.dir, f)
+          }
         })
         Object.assign(result.checks, kb.checks)
         result.boxes = { ...result.boxes, ...kb.boxes }
         keyboard = kb.keyboard
-        for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: `Focus ${s.n}: ${s.name}`, file: s.file, at: new Date().toISOString() })
-      }
-      // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
-      let view = null
-      if (htmlView) {
-        try {
-          // the same PII masks as the screenshot (#27), applied to the text of the copy
-          const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])] })
-          await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
-          view = { revision: rev.id, entry: rev.entry }
-        } catch (error) { view = { error: String(error.message).slice(0, 200) } }
+        for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: scrubText(redact(`Focus ${s.n}: ${s.name}`), mask.auto ? mask : null), file: s.file, at: new Date().toISOString() })
       }
       const record = {
         schema: 'qawalk.capture.v1',
@@ -254,6 +297,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...extra,
         http_status: extra.http_status ?? result.http_status ?? null,
         capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+        // what was hidden and where (#64): the canvas marks these places, so a blank is not read as a bug
+        ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: maskedPage?.counts || {}, rects: (maskedPage?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
         ...(ignore.length ? { ignore } : {}),
         ...(unstable ? { unstable } : {}),
         ...(keyboard ? { keyboard } : {}),
@@ -261,31 +306,65 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
         files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }
       }
+      // the whole record goes through the mask: remembered values, and with detection anything that looks like
+      // personal data (console lines, blocked requests, the URL, evidence, metadata)
+      const stored = mask.auto ? scrubDeep(redactDeep(record), mask, RECORD_PLAIN_KEYS) : redactDeep(record)
+      if (mask.active) await saveKnownPii(run.dir)
       const file = paths.step(stepId, locale, viewport, 'capture.json')
-      await writeJson(file, redactDeep(record))
+      await writeJson(file, stored)
       await addCapture(run.dir, { type: 'capture', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, file: relative(run.dir, file), at: record.captured_at })
-      results.push(record)
-    } finally { await undoVariant() }
+      results.push(stored)
+    } finally {
+      // the page gets its own text back, also when the capture failed half-way
+      if (mask.active) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {})
+      await undoVariant()
+    }
   }
   const last = page.viewportSize()
   if (original && (!last || last.width !== original.width || last.height !== original.height)) await page.setViewportSize(original)
   return results
 }
 
-export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0 }) {
+/**
+ * A focus shot of the keyboard walk with the mask on: the page masked for the moment of the shot, boxes drawn over
+ * the masked places in the clip, the page restored (the focus stays where the walk put it).
+ */
+async function maskedShot (page, mask, clip) {
+  try {
+    const masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
+    rememberPii(masking.pairs)
+    const png = await page.screenshot({ clip, scale: 'css', animations: 'disabled', caret: 'hide' })
+    const doc = await page.evaluate(() => ({ x: scrollX, y: scrollY }))
+    const boxes = masking.rects.filter((r) => r.box).map((r) => ({ ...r, x: r.x - doc.x - clip.x, y: r.y - doc.y - clip.y }))
+    return boxes.length ? (await encodeImage(page, png, { format: 'png', masks: boxes })).image : png
+  } finally { await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
+}
+
+export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0, options = null }) {
   const paths = runPaths(run.dir)
   const el = page.locator(selector).first()
   const name = `${node.id}-${locale}-${Date.now().toString(36)}.png`
   const file = paths.attachment(name)
   await mkdir(dirname(file), { recursive: true })
-  if (padding > 0) {
-    // element plus a margin of surrounding page, clipped from a full-page shot (documentation crops)
+  // the same mask as the step's capture (#64): masked text in the page, boxes over masked elements
+  const opts = options || captureOptions(null, node)
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  let masking = null
+  try {
+    if (mask.active) { await loadKnownPii(run.dir); masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() })); rememberPii(masking.pairs); await saveKnownPii(run.dir) }
+    const boxes = (masking?.rects || []).filter((r) => r.box)
+    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
     const box = await el.boundingBox()
     if (!box) throw new Error(`crop: ${selector} is not visible`)
-    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
+    // element plus a margin of surrounding page (documentation crops), clipped from a full-page shot
     const x = Math.max(0, box.x + doc.x - padding); const y = Math.max(0, box.y + doc.y - padding)
-    await page.screenshot({ path: file, fullPage: true, clip: { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) } })
-  } else await el.screenshot({ path: file })
+    const clip = { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) }
+    const png = padding > 0 || boxes.length ? await page.screenshot({ fullPage: true, clip }) : await el.screenshot()
+    if (boxes.length) {
+      const enc = await encodeImage(page, png, { format: 'png', masks: boxes.map((r) => ({ ...r, x: r.x - clip.x, y: r.y - clip.y })) })
+      await writeFile(file, enc.image)
+    } else await writeFile(file, png)
+  } finally { if (mask.active) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
   const entry = { type: 'attachment', kind: 'crop', node: node.id, locale, title: title || selector, selector, file: relative(run.dir, file), viewport: page.viewportSize(), at: new Date().toISOString() }
   await addCapture(run.dir, entry)
   return entry

package/src/lib/evaluate.mjs

@@ -4,6 +4,8 @@ import { readCaptures, runPaths } from './run.mjs'
 import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
 import { visionSettings, visionCriteria, templateReason, VISION_PROMPT_VERSION } from './vision-core.mjs'
 import { prepareImages, visionPlan, askVision, hostedVisionSender } from './vision.mjs'
+import { triageSettings, probeFindings, TRIAGE_PROMPT_VERSION } from './triage-core.mjs'
+import { askTriage, hostedTriageSender, chooseFrames, frameBatches, mergeFindings } from './triage.mjs'
 import { KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
 import { packHelp } from './pack-help.mjs'
@@ -31,7 +33,7 @@ export const evaluationSource = (node, capture) => sha256({ capture, criteria: n
  * asked, the vision evaluator. Every semantic verdict with a score carries a one-sentence `reason`: the vision
  * model's own, or a sentence saying what the text evaluator read (it returns a probability only).
  */
-export function combine ({ node, capture, jev, vision = null, thresholds, environment = null }) {
+export function combine ({ node, capture, jev, vision = null, triage = null, cascade = false, thresholds, environment = null }) {
   return (node.criteria || []).map((c) => {
     if (!appliesOn(c, environment)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies on ${c.environments.join(', ')} only; this walk ran on ${environment}.`, na: true, not_applicable: 'environment' }
     if (!appliesToState(c, capture?.step_state)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies to the state${c.states.length > 1 ? 's' : ''} ${c.states.join(', ')} only; this capture shows ${capture?.step_state || 'default'}.`, na: true, not_applicable: 'state' }
@@ -48,15 +50,19 @@ export function combine ({ node, capture, jev, vision = null, thresholds, enviro
       const answer = jev?.response?.answers?.[c.id]
       const text = answer?.noul ?? null
       const seen = vision?.answers?.[c.id] || null
-      const score = seen ? seen.probability : text
+      // Clef's answer on the screenshot (#71): recorded beside the others; with cascade on it decides when it is sure
+      const clef = triage?.criteria?.[c.id] ?? null
+      const clefDecides = cascade && !seen && clef != null && (clef <= thresholds.fail || clef >= thresholds.pass)
+      const score = seen ? seen.probability : clefDecides ? clef : text
       const verdict = verdictFor(score, thresholds)
-      const reason = seen?.reason || templateReason({ score, scope: c.check.scope, about: c.check.about, verdict })
+      const reason = seen?.reason || (clefDecides ? `Judged from the screenshot by Clef: ${Math.round(clef * 100)} % yes, clear enough for a verdict.` : templateReason({ score, scope: c.check.scope, about: c.check.about, verdict }))
       return {
         id: c.id, type: 'semantic', verdict, evidence: null, score,
-        provider: seen ? `${vision.provider || 'anthropic'}-vision` : answer ? 'typesafe-jev' : null,
+        provider: seen ? `${vision.provider || 'anthropic'}-vision` : clefDecides ? 'cloudflare-clef' : answer ? 'typesafe-jev' : null,
         note: score == null ? (jev?.skipped || jev?.error || vision?.skipped || vision?.error || null) : null,
         ...(reason ? { reason, reason_source: seen?.reason ? 'model' : 'template' } : {}),
-        ...(seen && text != null ? { text_score: text } : {}),
+        ...((seen || clefDecides) && text != null ? { text_score: text } : {}),
+        ...(clef != null ? { clef_score: clef } : {}),
         ...(!seen && (vision?.skipped || vision?.error) && vision.ids?.includes(c.id) ? { vision_note: vision.skipped || vision.error } : {})
       }
     }
@@ -80,12 +86,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   // with an own text-evaluator key the vision evaluator uses an own key too, unless hosted: true says otherwise
   if (vset && process.env[settings.api_key_env] && config.evaluators?.vision?.hosted !== true) vset.hosted = false
   const vision = vset ? visionContext({ config, root, settings: vset, report, fetchImpl }) : null
+  // triage (#65, experimental): the kind of problem on each screen and in recording frames, advisory only
+  const tset = triageSettings(config)
+  if (tset) report.triage = { model: tset.model, screens: 0, frames: 0, calls: 0, cached: 0, findings: 0, shown: 0, skipped: null, via: {}, ms: [] }
+  const triage = tset ? triageContext({ config, root, settings: tset, report, fetchImpl, uncertain: (vset || {}).uncertain, thresholds: settings.thresholds }) : null
   // checks against the design reference and the token set (pack design, #42) need the service and a browser
   const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
   // style checks against a token (style_equals token: color.brand.primary) use the project's token set
   const styleTokens = scenario.nodes.some((n) => tokenCriteria(n).length) ? (drift ? { tokens: drift.tokens, mode: drift.mode } : await runTokens({ config, root })) : null
   try {
-  const latest = latestCaptures(await readCaptures(run.dir))
+  const entries = await readCaptures(run.dir)
+  const latest = latestCaptures(entries)
   // stack steps: semantic criteria only on the first N pages (items.semantic_sample, default 20); cost stays bounded
   const sampled = new Map()
   for (const e of latest) {
@@ -129,13 +140,15 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
       }
     }))
-    // second opinion with the screenshot for flagged and uncertain criteria
+    // what kind of problem each screen and its recording shows; with cascade on, Clef may settle uncertain criteria
+    if (triage) await triage.ask(work.filter((w) => w.current && !w.outOfSample), run.dir, entries)
+    // second opinion with the screenshot for flagged and uncertain criteria (those Clef did not settle)
     if (vision) await vision.ask(work.filter((w) => w.current && !w.outOfSample && w.request), run.dir)
-    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen } of work) {
+    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen, triage: tri } of work) {
       const source = evaluationSource(node, capture)
       if (drift && current && needsDrift(node)) { capture.checks = { ...capture.checks, ...await driftForCapture({ node, entry, capture, runDir: run.dir, ctx: drift }) }; report.drift++ }
       if (styleTokens?.tokens && current && tokenCriteria(node).length) capture.checks = { ...capture.checks, ...storedValueChecks(node, capture, { ...styleTokens, recorded: !!capture.rechecked_at, only: (c) => !!c.check.token && STYLE_ASSERTS.has(c.check.assert) }) }
-      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, thresholds: settings.thresholds, environment })
+      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, triage: tri, cascade: !!tset?.cascade, thresholds: settings.thresholds, environment })
       // pages outside the semantic sample carry only the deterministic results
       if (outOfSample) criteria = criteria.filter((c) => c.type !== 'semantic' || c.evidence === 'no console errors')
       const evaluation = {
@@ -156,6 +169,8 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         thresholds: settings.thresholds,
         state_chars: request ? JSON.stringify(request.state).length : null,
         ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
+        // advisory findings (#65, #66): the kind of problem, never a verdict
+        ...(tri ? { triage: tri.findings ? { provider: 'cloudflare-clef', via: tri.via, model: tri.model, prompt_version: TRIAGE_PROMPT_VERSION, requests: tri.requests, cached: tri.cached, ms: tri.ms } : { provider: 'cloudflare-clef', skipped: tri.skipped || null, error: tri.error || null }, ...(tri.findings?.length ? { findings: tri.findings } : {}) } : {}),
         ...(seen ? { vision: seen.answers ? { provider: seen.provider, via: seen.via, model: seen.model, prompt_version: VISION_PROMPT_VERSION, cached: seen.cached, usage: seen.usage || null, answers: seen.answers } : { provider: seen.provider, skipped: seen.skipped || null, error: seen.error || null } } : {}),
         criteria
       }
@@ -165,8 +180,11 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
       for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes${c.reason_source === 'model' ? `: ${c.reason}` : ''}` : c.note) || null, ...(c.reason ? { reason: c.reason } : {}), ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
     }
   }
-  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close() }
-  const { hostedStop, visionStop, ...out } = report
+  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close(); await triage?.close() }
+  // triage is on by default but stays silent where nobody asked for it and it did not run (experiment off, no service)
+  if (report.triage && !report.triage.screens && config.evaluators?.triage == null) delete report.triage
+  if (report.triage) { const ms = report.triage.ms.filter((x) => x != null).sort((a, b) => a - b); report.triage.median_ms = ms.length ? ms[Math.floor(ms.length / 2)] : null; delete report.triage.ms }
+  const { hostedStop, visionStop, triageStop, ...out } = report
   return { ...out, outdated: [...report.outdated] }
 }
 
@@ -249,7 +267,7 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
       const queue = []
       for (const w of work) {
         const semantic = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && w.request.questions[c.id])
-        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings })
+        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings }).filter((c) => !w.triageDecided?.has(c.id))
         if (crit.length) queue.push({ w, crit })
       }
       await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 3, queue.length)) }, async () => {
@@ -281,6 +299,91 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
   }
 }
 
+/**
+ * The triage step of an evaluation (#65): one Clef request per screen (page tiles), and for a step with a recording
+ * up to `frames_per_step` chosen frames in requests of four. Uncertain semantic criteria ride along as extra
+ * questions (#71). Fills `w.triage` with merged findings; a stopped service (experiment off, allowance) ends triage
+ * for this evaluation with one note in the report.
+ */
+function triageContext ({ config, root, settings, report, fetchImpl, uncertain = [0.15, 0.85], thresholds = { pass: 0.85, fail: 0.15 } }) {
+  const cacheDir = join(resolve(root, config.output), '.cache', 'triage')
+  const hosted = hostedTriageSender(config, report)
+  let browser = null; let pagePromise = null
+  const page = () => (pagePromise ||= (async () => {
+    const { launchEphemeral } = await import('./browser.mjs')
+    browser = await launchEphemeral(config)
+    const p = await browser.newPage()
+    await p.setContent('<!doctype html><title>triage</title>')
+    return p
+  })())
+  const allowCall = () => { if (report.triage.calls >= (settings.max_calls ?? 200)) return false; report.triage.calls++; return true }
+  // a Cloudflare key in the environment (a CI that deploys with wrangler) is used only when triage was asked for in
+  // the config: on by default means through the service, where the organization switched the experiment on
+  const own = config.evaluators?.triage != null && process.env.CLOUDFLARE_API_TOKEN && process.env.CLOUDFLARE_ACCOUNT_ID
+  const unavailable = !own && !process.env[config.share?.token_env || 'KALOKO_TOKEN'] ? 'no service token (KALOKO_TOKEN)' : null
+  let framesLeft = settings.frames_per_run
+  const ask = async (w, payload, where) => {
+    const r = await askTriage({ payload, settings, cacheDir, where, fetchImpl, hosted, allowCall })
+    if (r.findings) { if (r.cached) report.triage.cached++; report.triage.via[r.via] = (report.triage.via[r.via] || 0) + 1; report.triage.ms.push(r.ms) }
+    else if (!report.triage.skipped) report.triage.skipped = r.skipped || r.error
+    return r
+  }
+  return {
+    async ask (work, runDir, entries) {
+      if (unavailable) { report.triage.skipped = unavailable; return }
+      if (!own && !(await hosted.ready())) { report.triage.skipped = report.triageStop; return }
+      const queue = [...work]
+      await Promise.all(Array.from({ length: Math.max(1, Math.min(3, queue.length)) }, async () => {
+        for (let w = queue.shift(); w; w = queue.shift()) {
+          if (report.triageStop) break
+          const file = w.capture.files?.image || w.capture.files?.png
+          if (!file) continue
+          try {
+            const [lo, hi] = uncertain
+            const criteria = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && c.check.about !== 'console' && w.request?.questions?.[c.id])
+              .filter((c) => { const s = w.jev?.response?.answers?.[c.id]?.noul; return s != null && s > lo && s < hi })
+              .map((c) => ({ id: c.id, question: c.check.question }))
+            const context = { purpose: purposeOf(w.node, w.capture), url: w.capture.url, locale: w.capture.locale, viewport: w.entry.viewport, signals: { overflow: (w.capture.overflow || []).length, console_errors: (w.capture.console_errors || []).length } }
+            const { images } = await prepareImages({ page: await page(), file: join(runDir, file), cssWidth: w.capture.size?.width, full: true, settings: { max_width: settings.max_width, tile_height: settings.tile_height, max_tiles: settings.max_tiles } })
+            const tiles = images.slice(0, settings.max_tiles || 2).map((i) => `data:${i.media_type};base64,${i.data}`)
+            report.triage.screens++
+            const lists = []; const results = []
+            const screen = await ask(w, { kind: 'screen', context, criteria, images: tiles }, {})
+            results.push(screen)
+            if (screen.findings) lists.push(screen.findings)
+            // the recording of this capture (same walk), as frames
+            const rec = entries.filter((e) => e.type === 'recording' && e.node === w.entry.node && e.locale === w.entry.locale && e.viewport === w.entry.viewport && e.captured_at && e.captured_at === w.capture.captured_at).pop()
+            if (rec && framesLeft > 0 && settings.frames_per_step > 0 && !report.triageStop) {
+              const side = await readJson(join(runDir, rec.file), null)
+              const video = side?.files?.webm || side?.files?.mp4
+              // a recording that cannot be read costs its frames only, never the screen's findings
+              if (video) try {
+                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.frame_width || 640 })
+                framesLeft -= frames.length; report.triage.frames += frames.length
+                for (const batch of frameBatches(frames)) {
+                  const r = await ask(w, { kind: 'motion', context: { ...context, frames: batch.map((f) => ({ t_ms: f.t, why: f.why })) }, images: batch.map((f) => f.data) }, { frames: batch.map((f) => f.t) })
+                  results.push(r)
+                  if (r.findings) lists.push(r.findings)
+                }
+              } catch (error) { report.triage.frameErrors = (report.triage.frameErrors || 0) + 1 }
+            }
+            const done = results.filter((r) => r.findings)
+            if (!done.length) { w.triage = { skipped: results[0]?.skipped || null, error: results[0]?.error || null }; continue }
+            const findings = mergeFindings([...lists, probeFindings(w.capture)])
+            report.triage.findings += findings.length; report.triage.shown += findings.filter((f) => f.shown).length
+            w.triage = { findings, criteria: screen.criteria || {}, model: done[0].model, via: done[0].via, cached: done.every((r) => r.cached), requests: done.length, ms: done.reduce((n, r) => n + (r.ms || 0), 0) || null }
+            // cascade (#71): criteria Clef is sure about skip the vision evaluator
+            if (settings.cascade) w.triageDecided = new Set(Object.entries(screen.criteria || {}).filter(([, p]) => p <= thresholds.fail || p >= thresholds.pass).map(([id]) => id))
+          } catch (error) {
+            w.triage = { error: String(error.message || error).slice(0, 300) }
+          }
+        }
+      }))
+    },
+    async close () { if (browser) await browser.close().catch(() => {}) }
+  }
+}
+
 /**
  * Sends a JEV request through the organization's service (hosted evaluation) when a service token is set.
  * An allowance used up or a service without hosting stops further hosted calls for this run with one message.

package/src/lib/htmlview.mjs

@@ -68,7 +68,7 @@ export function viewInPage ({ mask = [] }) {
  * cookies and gate headers as the capture), rewrites CSS url()s, writes the bundle into the run directory
  * (live/<revision>/…) and returns the revision ({ id, entry, files, sizes }).
  */
-export async function buildHtmlView ({ page, run, scenarioId, node, locale, viewport, mask = [] }) {
+export async function buildHtmlView ({ page, run, scenarioId, node, locale, viewport, mask = [], scrub = null }) {
   const { html, assets } = await page.evaluate(viewInPage, { mask })
   const request = page.context().request
   const files = {}, sizes = {}, bodies = new Map()
@@ -109,6 +109,8 @@ export async function buildHtmlView ({ page, run, scenarioId, node, locale, view
     const name = await fetchAsset(url)
     out = out.split(`__QW_ASSET_${i}__`).join(name || 'about:blank')
   }
+  // personal data (#64): attributes masked only for storage, and the values remembered for the run
+  if (scrub) out = scrub(out)
   const htmlBody = Buffer.from(out, 'utf8')
   files['index.html'] = await sha256Hex(htmlBody); sizes['index.html'] = htmlBody.length; bodies.set('index.html', htmlBody)
   const problems = bundleProblems(Object.keys(files).map((p) => ({ path: p, bytes: sizes[p] })), 'index.html')

package/src/lib/keyboard.mjs

@@ -135,7 +135,7 @@ const label = (s) => `${s.desc}${s.name ? ` "${s.name.slice(0, 30)}"` : ''}`
 /**
  * Walks the page with Tab and fills the keyboard criteria of `criteria` ({ id, check }).
  * opts: maxStops (60), focusChecks (stops checked for a visible indicator, 15), shots (focus screenshots, 5),
- * saveShot(n, pngBuffer) → relative file path or null.
+ * saveShot(n, pngBuffer, clip) → relative file path or null.
  * @returns {{ checks: Record<string, {ok, evidence}>, boxes: Record<string, Array>, keyboard: object }}
  */
 export async function keyboardChecks (page, criteria, { maxStops = 60, focusChecks = 15, shots = 5, saveShot = null } = {}) {
@@ -232,7 +232,9 @@ export async function keyboardChecks (page, criteria, { maxStops = 60, focusChec
           if (clip.width < 2 || clip.height < 2) continue
           const focused = await page.screenshot({ clip, scale: 'css', animations: 'disabled', caret: 'hide' })
           if (keyboard.shots.length < shots && saveShot) {
-            const file = await saveShot(n, await page.screenshot({ clip: clipOf(48), scale: 'css', animations: 'disabled', caret: 'hide' })).catch(() => null)
+            // the clip goes along: a masked capture takes its own shot of it with personal data covered (#64)
+            const shotClip = clipOf(48)
+            const file = await saveShot(n, await page.screenshot({ clip: shotClip, scale: 'css', animations: 'disabled', caret: 'hide' }), shotClip).catch(() => null)
             if (file) keyboard.shots.push({ n, file, name: s.name || s.desc })
           }
           if (!want.has('a11y_focus_visible') || verdicts.size >= focusChecks) continue

package/src/lib/mail.mjs

@@ -1,5 +1,5 @@
 // E-mails of a flow, from wherever the project's stack puts them. Every source returns the same
-// message shape: { id, subject, from, to: [address], date, html, text }.
+// message shape: { id, subject, from, to: [address], cc: [address], date, html, text }.
 //   mailpit — built in (REST API), optional gate auth;
 //   script  — a project module for anything else (letter_opener, Mailosaur, IMAP, the app's own API);
 //   file    — `kaloko mail --file`: a message the agent obtained itself (.eml, .html, .json, .txt).
@@ -40,7 +40,7 @@ export function mailpit (mail) {
         hitId = hit.ID
       }
       const m = await get(`/api/v1/message/${encodeURIComponent(hitId)}`)
-      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
+      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), cc: (m.Cc || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
     }
   }
 }
@@ -95,6 +95,7 @@ export function normalize (m) {
     subject: m.subject ?? m.Subject ?? '',
     from: typeof from === 'object' && from ? address(from) : from || '',
     to: list(m.to ?? (m.To || []).map?.((t) => t.Address ?? t)),
+    cc: list(m.cc ?? (m.Cc || []).map?.((t) => t.Address ?? t)),
     date: m.date ?? m.Date ?? null,
     html: m.html ?? m.HTML ?? null,
     text: m.text ?? m.Text ?? null
@@ -109,8 +110,8 @@ export async function readMessageFile (file) {
   if (ext === '.eml' || ext === '.msg' || ext === '.mbox') return parseEml(raw)
   if (ext === '.json') return normalize(JSON.parse(raw.toString('utf8')))
   const body = raw.toString('utf8')
-  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], date: null, html: body, text: null }
-  return { id: '', subject: '', from: '', to: [], date: null, html: null, text: body }
+  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], cc: [], date: null, html: body, text: null }
+  return { id: '', subject: '', from: '', to: [], cc: [], date: null, html: null, text: body }
 }
 
 // --- RFC 5322 / MIME, enough for what mailers send: multipart, base64, quoted-printable, charsets ---
@@ -175,6 +176,7 @@ export function parseEml (raw) {
     subject: h('subject'),
     from: h('from'),
     to: list(h('to')),
+    cc: list(h('cc')),
     date: top.headers.date ? new Date(top.headers.date).toISOString() : null,
     html: found.html ?? null,
     text: found.text ?? null