kaloko 0.12.0 → 0.12.1

package/README.md

@@ -420,9 +420,11 @@ same length in the HTML and the page outline, and the same places in traces,
 recordings and captured e-mails. `pii: redact` on a scenario finds e-mails,
 phone numbers, birth numbers, IBANs and addresses by itself; `pii:
 pseudonymize` puts consistent fake values in their place instead. An
-environment with `pii: required` gets no run without it, and `npx kaloko share`
-refuses a run that still holds an unmasked e-mail or phone number, naming the
-step and element (`--allow-pii` overrides). Checks always read the real page.
+environment with `pii: required` gets no run without it. When such a run still
+holds an unmasked e-mail or phone number, `npx kaloko share` refuses it and
+names the step and element (`--allow-pii` overrides); other runs are shared
+with a warning. The site's own contacts (its domain, the footer, Organization
+JSON-LD) do not count. Checks always read the real page.
 
 Set `browser.channel: chrome` to use an installed Chrome, or run
 `npx kaloko browser install` for the Chromium of kaloko's Playwright

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.12.0",
+  "version": "0.12.1",
   "description": "See what your AI agents built: every screen, language and device walked, assessed and approved on one canvas.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -90,9 +90,10 @@
                 "model": { "enum": ["clef", "clef-flash"], "default": "clef", "description": "clef found more on the first bench; clef-flash answers about twice as fast." },
                 "keep": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.7, "description": "Findings at or above this probability show on the canvas; lower ones fold under Less likely." },
                 "maybe": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.5, "description": "Findings below this probability are dropped." },
-                "max_width": { "type": "integer", "minimum": 320, "maximum": 2048, "default": 1024 },
-                "tile_height": { "type": "integer", "minimum": 400, "maximum": 4000, "default": 1400 },
-                "max_tiles": { "type": "integer", "minimum": 1, "maximum": 4, "default": 4 },
+                "max_width": { "type": "integer", "minimum": 320, "maximum": 2048, "default": 896 },
+                "tile_height": { "type": "integer", "minimum": 400, "maximum": 4000, "default": 1024, "description": "Clef reads about one token per 40 pixels within 65,536 tokens: keep max_tiles × max_width × tile_height near 1.8 million." },
+                "max_tiles": { "type": "integer", "minimum": 1, "maximum": 4, "default": 2 },
+                "frame_width": { "type": "integer", "minimum": 240, "maximum": 1280, "default": 640, "description": "Width of recording frames sent to Clef (four per request)." },
                 "frames_per_step": { "type": "integer", "minimum": 0, "maximum": 12, "default": 6, "description": "Frames chosen from a step's recording (around taps, sharp changes and the end); 0 skips recordings." },
                 "frames_per_run": { "type": "integer", "minimum": 0, "default": 60 },
                 "max_calls": { "type": "integer", "minimum": 0, "default": 200, "description": "Uncached triage requests per evaluation." },
@@ -129,7 +130,8 @@
       "additionalProperties": false,
       "description": "Personal data (#64).",
       "properties": {
-        "allow": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Values that look like personal data but are not, such as the company's contact e-mail or phone number. * matches any text (*@acme.example). Neither masked nor stopped by kaloko share." }
+        "allow": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Values that look like personal data but are not, such as the company's contact e-mail or phone number. * matches any text (*@acme.example). Neither masked nor stopped by kaloko share." },
+        "site_contacts": { "type": "boolean", "default": true, "description": "kaloko share takes the site's own contacts for not personal data: e-mails on the environment's domain, e-mails and phone numbers in the page footer, and those of an Organization or ContactPoint in JSON-LD. false checks them like any other value." }
       }
     },
     "share": {

package/skills/kaloko/reference/acceptance.md

@@ -165,8 +165,8 @@ Masks cover every output of the step: the screenshot gets a striped box, the HTM
 `[hidden]` of the same length, traces, recordings and captured e-mails are covered too; checks still read the real
 page. On data copied from production add `pii: redact` to the scenario (automatic detection of e-mails, phones,
 birth numbers, IBANs, addresses) or `pii: pseudonymize` (consistent fake values, readable screens). An environment
-with `pii: required` refuses a run without it, and `kaloko share` stops on an unmasked e-mail or phone number and
-names the step and element: mask it and capture again, list a company contact under `pii.allow` in the config, and
+with `pii: required` refuses a run without it, and in a run that hides personal data `kaloko share` stops on an
+unmasked e-mail or phone number and names the step and element (other runs are shared with a warning): mask it and capture again, list a company contact under `pii.allow` in the config, and
 use `--allow-pii` only when a person says the value may go up.
 
 ## Keeping plan and scenario aligned

package/src/cli.mjs

@@ -62,7 +62,7 @@ import { passesOf, enginesOf, contextOptionsOf, launchEngine, pickKeys, variantF
 import { startScope, scopeKeys, dimensionLine, baseViewports, engineKeys, appDeviceGroups } from './lib/walk-dimensions.mjs'
 import { deviceConfig, prepareAndroid, readyAndroid, finishAndroid, prepareSimulator, applyAppAppearance } from './lib/apps/devices.mjs'
 import { attachAndroidWebView } from './lib/apps/webview.mjs'
-import { piiSettings, piiFindings, guardAllow, formatFindings } from './lib/pii.mjs'
+import { piiSettings, piiFindings, piiRequested, guardAllow, formatFindings } from './lib/pii.mjs'
 
 
 const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
@@ -201,8 +201,9 @@ Results
                                     hash, so unchanged captures are not sent again and a broken-off share resumes;
                                     --pr adds or updates one comment on the branch's pull request via gh (KALOKO_PR in
                                     CI), or a note on the GitLab merge request (KALOKO_GITLAB_TOKEN)
-       [--allow-pii]                An unmasked e-mail, phone number, birth number or IBAN in the run stops the upload
-                                    and names the step and element; --allow-pii uploads anyway
+       [--allow-pii]                In a run that hides personal data, an unmasked e-mail, phone number, birth number or
+                                    IBAN stops the upload and names the step and element; --allow-pii uploads anyway
+                                    (other runs upload with a warning)
   feedback [--run id] [--scenario id] [--json]
                                     Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies;
                                     returned steps first, with their notes
@@ -1744,7 +1745,7 @@ const commands = {
     const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
     say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
     if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
-    if (report.triage) say(report.triage.screens ? `Triage (experimental, ${report.triage.model}): ${report.triage.screens} screen(s)${report.triage.frames ? ` and ${report.triage.frames} recording frame(s)` : ''}, ${report.triage.shown} finding(s) to look at${report.triage.findings > report.triage.shown ? ` and ${report.triage.findings - report.triage.shown} less likely` : ''}; they never change a verdict.${report.triage.skipped ? ` Stopped early: ${report.triage.skipped}` : ''}` : `Triage (experimental): not run: ${report.triage.skipped}`)
+    if (report.triage) say(report.triage.screens ? `Triage (experimental, ${report.triage.model}): ${report.triage.screens} screen(s)${report.triage.frames ? ` and ${report.triage.frames} recording frame(s)` : ''}, ${report.triage.shown} finding(s) to look at${report.triage.findings > report.triage.shown ? ` and ${report.triage.findings - report.triage.shown} less likely` : ''}; they never change a verdict.${report.triage.skipped ? ` Not every request was answered: ${report.triage.skipped}` : ''}` : `Triage (experimental): not run${report.triage.skipped ? `: ${report.triage.skipped}` : ' (no screenshot to look at)'}`)
     if (report.skipped.length) {
       // one line per reason: "12 captures: TYPESAFE_API_KEY is not set …"
       const byReason = new Map()
@@ -1812,9 +1813,12 @@ const commands = {
     }
     // personal data (#64): nothing goes up with an unmasked e-mail, phone number, birth number or IBAN in it
     const views = manifest.nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((vps) => Object.values(vps).map((c) => c.view?.revision).filter(Boolean))).map((id) => `live/${id}/index.html`)
-    const findings = await piiFindings({ dir: run.dir, files: [...files, ...views], manifest, allow: guardAllow({ config, env, scenario, run }) })
-    if (findings.length && !args['allow-pii']) throw new UserError(`Not shared: the run holds ${findings.length} unmasked personal data value(s):\n${formatFindings(findings)}\nMask them (capture.mask in the scenario, or pii: redact for automatic detection) and capture the steps again. Values that are not personal data, such as a company contact, go under pii.allow in kaloko.config.yml. To share anyway: kaloko share --allow-pii.`)
-    if (findings.length) say(`Sharing ${findings.length} unmasked personal data value(s), as --allow-pii says:\n${formatFindings(findings, 5)}`)
+    // a run that asked for hiding personal data is refused; any other run is shared with a warning
+    const findings = await piiFindings({ dir: run.dir, files: [...files, ...views], manifest, allow: guardAllow({ config, env, scenario, run }), site: { url: run.meta.environment?.base_url || env.base_url, contacts: config.pii?.site_contacts !== false } })
+    const strict = piiRequested({ run, env, scenario })
+    if (findings.length && strict && !args['allow-pii']) throw new UserError(`Not shared: the run holds ${findings.length} unmasked personal data value(s):\n${formatFindings(findings)}\nMask them (capture.mask in the scenario, or pii: redact for automatic detection) and capture the steps again. Values that are not personal data, such as a company contact, go under pii.allow in kaloko.config.yml. To share anyway: kaloko share --allow-pii.`)
+    if (findings.length && strict) say(`Sharing ${findings.length} unmasked personal data value(s), as --allow-pii says:\n${formatFindings(findings, 5)}`)
+    else if (findings.length) say(`This run shows ${findings.length} value(s) that look like personal data:\n${formatFindings(findings, 5)}\nShared anyway, because the run does not hide personal data. To hide it, add pii: redact to the scenario or the environment; values that are not personal data go under pii.allow in kaloko.config.yml.`)
     const days = Number(args.expires) || config.share?.expires_days || 30
     // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
     let entries = await hashFiles(run.dir, [...files])

package/src/lib/capture.mjs

@@ -5,7 +5,7 @@ import { probe } from './probe.mjs'
 import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
-import { maskSpec, maskList, inPage, maskInPage, rememberPii, knownPii, maskMailHeaders } from './pii.mjs'
+import { maskSpec, maskList, inPage, maskInPage, rememberPii, knownPii, maskMailHeaders, maskHtml, scrubDeep, scrubText, updateMasks, loadKnownPii, saveKnownPii, RECORD_PLAIN_KEYS } from './pii.mjs'
 
 /**
  * Encodes a PNG screenshot inside the page (no native dependency): the image as WebP (or PNG, lossless) and a
@@ -134,6 +134,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
   let netResults = null
   // personal data (#64): what this capture masks; recipients of a captured e-mail are masked before the body
   const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  // values masked by an earlier process of this run (kaloko walk, then kaloko capture) are masked here too
+  if (mask.active) await loadKnownPii(run.dir)
   if (mask.active && extra.mail) extra = { ...extra, mail: maskMailHeaders(extra.mail, mask) }
   // batch E: the engine of this page, and per key its colour scheme, motion and display mode (run scope variants)
   const engine = engineOf(page)
@@ -161,16 +163,36 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       const ignore = await resolveIgnore(page, node, locale, viewport)
       // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the mask and the keyboard walk
       await liveValueChecks({ page, node, result, url: page.url(), tokens })
-      // One mask for every output (#64): from here to the end of the capture the page shows placeholders or fake
-      // values instead of personal data, so the screenshot, HTML, outline, spec, keyboard shots and HTML view agree
+      // the console is read before the mask: the page's own reaction to masked text never counts as its error
+      const url = page.url()
+      const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
+      const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
+      // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
+      // an error page's own status ("Failed to load resource: … 404" for the document itself) is not a console error
+      const ownStatus = (e) => result.http_status >= 400 && /^Failed to load resource: the server responded with a status of \d+/.test(e.text) && e.src && sameDocument(e.src, url)
+      const errors = consoleEntries.filter((e) => e.type !== 'blocked' && sameDocument(e.url, url) && !e.text.includes('ERR_BLOCKED_BY_CLIENT') && !ownStatus(e)).map((e) => e.text)
+      const blocked = consoleEntries.filter((e) => e.type === 'blocked').map((e) => `${e.text}`)
+      for (const c of node.criteria || []) {
+        if (c.check.type === 'deterministic' && c.check.assert === 'no_console_errors') {
+          result.checks[c.id] = collected === null
+            ? { ok: null, evidence: 'Console was not observed.' }
+            : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
+        }
+      }
+      // One mask for every output (#64): from here until the HTML view the page shows placeholders or fake values
+      // instead of personal data, so the screenshot, HTML, outline, spec and HTML view agree
       if (mask.active) {
         masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
         rememberPii(masking.pairs)
-        await page.evaluate((known) => window.__kalokoMask?.update({ known }), knownPii()).catch(() => {})
-        // the outline the evaluators read comes from the masked page; checks and boxes keep the first probe
-        if (masking.rects.length || masking.pairs.length || Object.keys(masking.counts).length) {
-          const again = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)).catch(() => null)
-          if (again) for (const k of ['state', 'scoped', 'overflow', 'meta', 'design']) if (again[k] !== undefined) result[k] = again[k]
+        await saveKnownPii(run.dir)
+        await updateMasks(page)
+        // the outline the evaluators read comes from the masked page; checks and boxes keep the first probe. When the
+        // masked page cannot be read, nothing is stored: the first probe holds the real data
+        if (masking.changed) {
+          let again = null
+          try { again = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)) } catch {}
+          if (!again) throw new Error(`capture: the masked page of ${node.id} could not be read again; nothing was stored so no personal data leaves in the page outline`)
+          for (const [k, v] of Object.entries(again)) if (k !== 'checks' && k !== 'boxes' && k !== 'http_status') result[k] = v
         }
       }
       const maskRects = (masking?.rects || []).filter((r) => r.box)
@@ -193,7 +215,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css' })
       }
       // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
-      await writeFile(html, redact(await page.content()))
+      await writeFile(html, redact(maskHtml(await page.content(), masking?.attrs)))
       // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
       if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens, scope?.variants?.[viewport]?.token_mode || scope?.variants?.[viewport]?.color_scheme || null)
       // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
@@ -217,27 +239,27 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         image = paths.step(stepId, locale, viewport, 'png'); thumbWritten = false
         await writeFile(image, shot)
       }
-      const url = page.url()
-      const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
-      const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
-      // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
-      // an error page's own status ("Failed to load resource: … 404" for the document itself) is not a console error
-      const ownStatus = (e) => result.http_status >= 400 && /^Failed to load resource: the server responded with a status of \d+/.test(e.text) && e.src && sameDocument(e.src, url)
-      const errors = consoleEntries.filter((e) => e.type !== 'blocked' && sameDocument(e.url, url) && !e.text.includes('ERR_BLOCKED_BY_CLIENT') && !ownStatus(e)).map((e) => e.text)
-      const blocked = consoleEntries.filter((e) => e.type === 'blocked').map((e) => `${e.text}`)
       // network checks (internal links, hreflang pairs) run once per page, outside the page, with a walk-wide cache
       const net = (node.criteria || []).filter((c) => c.check.type === 'deterministic' && NETWORK_ASSERTS.has(c.check.assert))
       if (net.length) {
         netResults ||= await networkChecks(page, net, url, { run, locale })
         Object.assign(result.checks, netResults)
       }
-      for (const c of node.criteria || []) {
-        if (c.check.type === 'deterministic' && c.check.assert === 'no_console_errors') {
-          result.checks[c.id] = collected === null
-            ? { ok: null, evidence: 'Console was not observed.' }
-            : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
-        }
+      // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
+      let view = null
+      if (htmlView) {
+        try {
+          // the page is still masked here (#64); the selectors are covered in the copy as well (#27)
+          const attrs = masking?.attrs
+          const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])], scrub: (text) => redact(maskHtml(text, attrs)) })
+          await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
+          view = { revision: rev.id, entry: rev.entry }
+        } catch (error) { view = { error: String(error.message).slice(0, 200) } }
       }
+      // the page gets its own text back before the keyboard walk: Tab and the accessibility tree see the real page,
+      // and only the focus shots are taken masked
+      const maskedPage = masking
+      if (masking) { await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}); masking = null }
       // keyboard and screen-reader checks (#19): a Tab walk and the accessibility tree, after the screenshot and after
       // the console was read (focus handlers of the page must not count as its errors)
       let keyboard = null
@@ -252,22 +274,16 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       }
       if (needsKeyboard(kbNode) && kbSampled) {
         const kb = await keyboardChecks(page, kbNode.criteria, {
-          saveShot: async (n, png) => { const f = paths.step(stepId, locale, viewport, `focus-${n}.png`); await writeFile(f, png); return relative(run.dir, f) }
+          saveShot: async (n, png, clip) => {
+            const f = paths.step(stepId, locale, viewport, `focus-${n}.png`)
+            await writeFile(f, mask.active ? await maskedShot(page, mask, clip) : png)
+            return relative(run.dir, f)
+          }
         })
         Object.assign(result.checks, kb.checks)
         result.boxes = { ...result.boxes, ...kb.boxes }
         keyboard = kb.keyboard
-        for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: `Focus ${s.n}: ${s.name}`, file: s.file, at: new Date().toISOString() })
-      }
-      // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
-      let view = null
-      if (htmlView) {
-        try {
-          // the page is still masked here (#64); the selectors are covered in the copy as well (#27)
-          const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])] })
-          await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
-          view = { revision: rev.id, entry: rev.entry }
-        } catch (error) { view = { error: String(error.message).slice(0, 200) } }
+        for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: scrubText(redact(`Focus ${s.n}: ${s.name}`), mask.auto ? mask : null), file: s.file, at: new Date().toISOString() })
       }
       const record = {
         schema: 'qawalk.capture.v1',
@@ -282,7 +298,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         http_status: extra.http_status ?? result.http_status ?? null,
         capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
         // what was hidden and where (#64): the canvas marks these places, so a blank is not read as a bug
-        ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: masking?.counts || {}, rects: (masking?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
+        ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: maskedPage?.counts || {}, rects: (maskedPage?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
         ...(ignore.length ? { ignore } : {}),
         ...(unstable ? { unstable } : {}),
         ...(keyboard ? { keyboard } : {}),
@@ -290,13 +306,17 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
         files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }
       }
+      // the whole record goes through the mask: remembered values, and with detection anything that looks like
+      // personal data (console lines, blocked requests, the URL, evidence, metadata)
+      const stored = mask.auto ? scrubDeep(redactDeep(record), mask, RECORD_PLAIN_KEYS) : redactDeep(record)
+      if (mask.active) await saveKnownPii(run.dir)
       const file = paths.step(stepId, locale, viewport, 'capture.json')
-      await writeJson(file, redactDeep(record))
+      await writeJson(file, stored)
       await addCapture(run.dir, { type: 'capture', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, file: relative(run.dir, file), at: record.captured_at })
-      results.push(record)
+      results.push(stored)
     } finally {
       // the page gets its own text back, also when the capture failed half-way
-      if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {})
+      if (mask.active) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {})
       await undoVariant()
     }
   }
@@ -305,6 +325,21 @@ export async function captureStep ({ page, node, locale, viewports, run, console
   return results
 }
 
+/**
+ * A focus shot of the keyboard walk with the mask on: the page masked for the moment of the shot, boxes drawn over
+ * the masked places in the clip, the page restored (the focus stays where the walk put it).
+ */
+async function maskedShot (page, mask, clip) {
+  try {
+    const masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
+    rememberPii(masking.pairs)
+    const png = await page.screenshot({ clip, scale: 'css', animations: 'disabled', caret: 'hide' })
+    const doc = await page.evaluate(() => ({ x: scrollX, y: scrollY }))
+    const boxes = masking.rects.filter((r) => r.box).map((r) => ({ ...r, x: r.x - doc.x - clip.x, y: r.y - doc.y - clip.y }))
+    return boxes.length ? (await encodeImage(page, png, { format: 'png', masks: boxes })).image : png
+  } finally { await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
+}
+
 export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0, options = null }) {
   const paths = runPaths(run.dir)
   const el = page.locator(selector).first()
@@ -316,7 +351,7 @@ export async function captureCrop ({ page, node, locale, selector, title, run, p
   const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
   let masking = null
   try {
-    if (mask.active) { masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() })); rememberPii(masking.pairs) }
+    if (mask.active) { await loadKnownPii(run.dir); masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() })); rememberPii(masking.pairs); await saveKnownPii(run.dir) }
     const boxes = (masking?.rects || []).filter((r) => r.box)
     const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
     const box = await el.boundingBox()
@@ -329,7 +364,7 @@ export async function captureCrop ({ page, node, locale, selector, title, run, p
       const enc = await encodeImage(page, png, { format: 'png', masks: boxes.map((r) => ({ ...r, x: r.x - clip.x, y: r.y - clip.y })) })
       await writeFile(file, enc.image)
     } else await writeFile(file, png)
-  } finally { if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
+  } finally { if (mask.active) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
   const entry = { type: 'attachment', kind: 'crop', node: node.id, locale, title: title || selector, selector, file: relative(run.dir, file), viewport: page.viewportSize(), at: new Date().toISOString() }
   await addCapture(run.dir, entry)
   return entry

package/src/lib/evaluate.mjs

@@ -89,7 +89,7 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   // triage (#65, experimental): the kind of problem on each screen and in recording frames, advisory only
   const tset = triageSettings(config)
   if (tset) report.triage = { model: tset.model, screens: 0, frames: 0, calls: 0, cached: 0, findings: 0, shown: 0, skipped: null, via: {}, ms: [] }
-  const triage = tset ? triageContext({ config, root, settings: tset, report, fetchImpl, uncertain: (vset || {}).uncertain }) : null
+  const triage = tset ? triageContext({ config, root, settings: tset, report, fetchImpl, uncertain: (vset || {}).uncertain, thresholds: settings.thresholds }) : null
   // checks against the design reference and the token set (pack design, #42) need the service and a browser
   const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
   // style checks against a token (style_equals token: color.brand.primary) use the project's token set
@@ -305,7 +305,7 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
  * questions (#71). Fills `w.triage` with merged findings; a stopped service (experiment off, allowance) ends triage
  * for this evaluation with one note in the report.
  */
-function triageContext ({ config, root, settings, report, fetchImpl, uncertain = [0.15, 0.85] }) {
+function triageContext ({ config, root, settings, report, fetchImpl, uncertain = [0.15, 0.85], thresholds = { pass: 0.85, fail: 0.15 } }) {
   const cacheDir = join(resolve(root, config.output), '.cache', 'triage')
   const hosted = hostedTriageSender(config, report)
   let browser = null; let pagePromise = null
@@ -317,7 +317,9 @@ function triageContext ({ config, root, settings, report, fetchImpl, uncertain =
     return p
   })())
   const allowCall = () => { if (report.triage.calls >= (settings.max_calls ?? 200)) return false; report.triage.calls++; return true }
-  const own = process.env.CLOUDFLARE_API_TOKEN && process.env.CLOUDFLARE_ACCOUNT_ID
+  // a Cloudflare key in the environment (a CI that deploys with wrangler) is used only when triage was asked for in
+  // the config: on by default means through the service, where the organization switched the experiment on
+  const own = config.evaluators?.triage != null && process.env.CLOUDFLARE_API_TOKEN && process.env.CLOUDFLARE_ACCOUNT_ID
   const unavailable = !own && !process.env[config.share?.token_env || 'KALOKO_TOKEN'] ? 'no service token (KALOKO_TOKEN)' : null
   let framesLeft = settings.frames_per_run
   const ask = async (w, payload, where) => {
@@ -343,7 +345,7 @@ function triageContext ({ config, root, settings, report, fetchImpl, uncertain =
               .map((c) => ({ id: c.id, question: c.check.question }))
             const context = { purpose: purposeOf(w.node, w.capture), url: w.capture.url, locale: w.capture.locale, viewport: w.entry.viewport, signals: { overflow: (w.capture.overflow || []).length, console_errors: (w.capture.console_errors || []).length } }
             const { images } = await prepareImages({ page: await page(), file: join(runDir, file), cssWidth: w.capture.size?.width, full: true, settings: { max_width: settings.max_width, tile_height: settings.tile_height, max_tiles: settings.max_tiles } })
-            const tiles = images.slice(0, 4).map((i) => `data:${i.media_type};base64,${i.data}`)
+            const tiles = images.slice(0, settings.max_tiles || 2).map((i) => `data:${i.media_type};base64,${i.data}`)
             report.triage.screens++
             const lists = []; const results = []
             const screen = await ask(w, { kind: 'screen', context, criteria, images: tiles }, {})
@@ -351,18 +353,19 @@ function triageContext ({ config, root, settings, report, fetchImpl, uncertain =
             if (screen.findings) lists.push(screen.findings)
             // the recording of this capture (same walk), as frames
             const rec = entries.filter((e) => e.type === 'recording' && e.node === w.entry.node && e.locale === w.entry.locale && e.viewport === w.entry.viewport && e.captured_at && e.captured_at === w.capture.captured_at).pop()
-            if (rec && framesLeft > 0 && !report.triageStop) {
+            if (rec && framesLeft > 0 && settings.frames_per_step > 0 && !report.triageStop) {
               const side = await readJson(join(runDir, rec.file), null)
               const video = side?.files?.webm || side?.files?.mp4
-              if (video) {
-                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.max_width })
+              // a recording that cannot be read costs its frames only, never the screen's findings
+              if (video) try {
+                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.frame_width || 640 })
                 framesLeft -= frames.length; report.triage.frames += frames.length
                 for (const batch of frameBatches(frames)) {
                   const r = await ask(w, { kind: 'motion', context: { ...context, frames: batch.map((f) => ({ t_ms: f.t, why: f.why })) }, images: batch.map((f) => f.data) }, { frames: batch.map((f) => f.t) })
                   results.push(r)
                   if (r.findings) lists.push(r.findings)
                 }
-              }
+              } catch (error) { report.triage.frameErrors = (report.triage.frameErrors || 0) + 1 }
             }
             const done = results.filter((r) => r.findings)
             if (!done.length) { w.triage = { skipped: results[0]?.skipped || null, error: results[0]?.error || null }; continue }
@@ -370,7 +373,7 @@ function triageContext ({ config, root, settings, report, fetchImpl, uncertain =
             report.triage.findings += findings.length; report.triage.shown += findings.filter((f) => f.shown).length
             w.triage = { findings, criteria: screen.criteria || {}, model: done[0].model, via: done[0].via, cached: done.every((r) => r.cached), requests: done.length, ms: done.reduce((n, r) => n + (r.ms || 0), 0) || null }
             // cascade (#71): criteria Clef is sure about skip the vision evaluator
-            if (settings.cascade) w.triageDecided = new Set(Object.entries(screen.criteria || {}).filter(([, p]) => p <= 0.15 || p >= 0.85).map(([id]) => id))
+            if (settings.cascade) w.triageDecided = new Set(Object.entries(screen.criteria || {}).filter(([, p]) => p <= thresholds.fail || p >= thresholds.pass).map(([id]) => id))
           } catch (error) {
             w.triage = { error: String(error.message || error).slice(0, 300) }
           }

package/src/lib/htmlview.mjs

@@ -68,7 +68,7 @@ export function viewInPage ({ mask = [] }) {
  * cookies and gate headers as the capture), rewrites CSS url()s, writes the bundle into the run directory
  * (live/<revision>/…) and returns the revision ({ id, entry, files, sizes }).
  */
-export async function buildHtmlView ({ page, run, scenarioId, node, locale, viewport, mask = [] }) {
+export async function buildHtmlView ({ page, run, scenarioId, node, locale, viewport, mask = [], scrub = null }) {
   const { html, assets } = await page.evaluate(viewInPage, { mask })
   const request = page.context().request
   const files = {}, sizes = {}, bodies = new Map()
@@ -109,6 +109,8 @@ export async function buildHtmlView ({ page, run, scenarioId, node, locale, view
     const name = await fetchAsset(url)
     out = out.split(`__QW_ASSET_${i}__`).join(name || 'about:blank')
   }
+  // personal data (#64): attributes masked only for storage, and the values remembered for the run
+  if (scrub) out = scrub(out)
   const htmlBody = Buffer.from(out, 'utf8')
   files['index.html'] = await sha256Hex(htmlBody); sizes['index.html'] = htmlBody.length; bodies.set('index.html', htmlBody)
   const problems = bundleProblems(Object.keys(files).map((p) => ({ path: p, bytes: sizes[p] })), 'index.html')

package/src/lib/keyboard.mjs

@@ -135,7 +135,7 @@ const label = (s) => `${s.desc}${s.name ? ` "${s.name.slice(0, 30)}"` : ''}`
 /**
  * Walks the page with Tab and fills the keyboard criteria of `criteria` ({ id, check }).
  * opts: maxStops (60), focusChecks (stops checked for a visible indicator, 15), shots (focus screenshots, 5),
- * saveShot(n, pngBuffer) → relative file path or null.
+ * saveShot(n, pngBuffer, clip) → relative file path or null.
  * @returns {{ checks: Record<string, {ok, evidence}>, boxes: Record<string, Array>, keyboard: object }}
  */
 export async function keyboardChecks (page, criteria, { maxStops = 60, focusChecks = 15, shots = 5, saveShot = null } = {}) {
@@ -232,7 +232,9 @@ export async function keyboardChecks (page, criteria, { maxStops = 60, focusChec
           if (clip.width < 2 || clip.height < 2) continue
           const focused = await page.screenshot({ clip, scale: 'css', animations: 'disabled', caret: 'hide' })
           if (keyboard.shots.length < shots && saveShot) {
-            const file = await saveShot(n, await page.screenshot({ clip: clipOf(48), scale: 'css', animations: 'disabled', caret: 'hide' })).catch(() => null)
+            // the clip goes along: a masked capture takes its own shot of it with personal data covered (#64)
+            const shotClip = clipOf(48)
+            const file = await saveShot(n, await page.screenshot({ clip: shotClip, scale: 'css', animations: 'disabled', caret: 'hide' }), shotClip).catch(() => null)
             if (file) keyboard.shots.push({ n, file, name: s.name || s.desc })
           }
           if (!want.has('a11y_focus_visible') || verdicts.size >= focusChecks) continue