kaloko 0.11.0 → 0.12.0

package/README.md

@@ -219,6 +219,10 @@ Every AI verdict comes with one sentence saying why. Criteria marked
 `vision: true`, and answers the text evaluator is unsure about, get a second
 look at the screenshot from a vision model (`ANTHROPIC_API_KEY`, or through
 Kaloko); `kaloko build` lists what to act on with the score and the reason.
+As an experiment you can switch on in Settings, Clef on Cloudflare also names
+the kind of problem each screen and its recording show (overlap, cut-off text,
+a raw translation key, an endless spinner); reviewers turn the right findings
+into comments with one click, and findings never change a verdict.
 The `a11y` pack walks the page with Tab (order, visible focus, traps, skip
 link) and reads names from the accessibility tree. Pixel diffs skip the
 regions a step ignores; `kaloko walk --stability 3` and `kaloko stability`
@@ -410,6 +414,16 @@ captured HTML, page text and console output.
 `npx kaloko doctor` checks that every variable the config names is set and that
 each environment lets you in.
 
+**Personal data.** `capture.mask` (CSS selectors) hides personal data in
+everything a step stores: a striped box on the screenshot, `[hidden]` of the
+same length in the HTML and the page outline, and the same places in traces,
+recordings and captured e-mails. `pii: redact` on a scenario finds e-mails,
+phone numbers, birth numbers, IBANs and addresses by itself; `pii:
+pseudonymize` puts consistent fake values in their place instead. An
+environment with `pii: required` gets no run without it, and `npx kaloko share`
+refuses a run that still holds an unmasked e-mail or phone number, naming the
+step and element (`--allow-pii` overrides). Checks always read the real page.
+
 Set `browser.channel: chrome` to use an installed Chrome, or run
 `npx kaloko browser install` for the Chromium of kaloko's Playwright
 (`KALOKO_BROWSER_CHANNEL=chromium` then uses it, which is how CI runs).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "See what your AI agents built: every screen, language and device walked, assessed and approved on one canvas.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -78,6 +78,28 @@
               }
             }
           ]
+        },
+        "triage": {
+          "description": "Triage (#65, experimental): Cloudflare's Clef names the kind of problem on each evaluated screen and in chosen frames of its recording (overlap, cut-off text, raw key, endless spinner…). Findings are advisory and never change a verdict. Runs through the organization's service once an admin switches the experiment on (Settings → Security → Experiments), or with CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID. false turns it off.",
+          "oneOf": [
+            { "const": false },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "model": { "enum": ["clef", "clef-flash"], "default": "clef", "description": "clef found more on the first bench; clef-flash answers about twice as fast." },
+                "keep": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.7, "description": "Findings at or above this probability show on the canvas; lower ones fold under Less likely." },
+                "maybe": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.5, "description": "Findings below this probability are dropped." },
+                "max_width": { "type": "integer", "minimum": 320, "maximum": 2048, "default": 1024 },
+                "tile_height": { "type": "integer", "minimum": 400, "maximum": 4000, "default": 1400 },
+                "max_tiles": { "type": "integer", "minimum": 1, "maximum": 4, "default": 4 },
+                "frames_per_step": { "type": "integer", "minimum": 0, "maximum": 12, "default": 6, "description": "Frames chosen from a step's recording (around taps, sharp changes and the end); 0 skips recordings." },
+                "frames_per_run": { "type": "integer", "minimum": 0, "default": 60 },
+                "max_calls": { "type": "integer", "minimum": 0, "default": 200, "description": "Uncached triage requests per evaluation." },
+                "cascade": { "type": "boolean", "default": false, "description": "Let Clef settle semantic criteria the text evaluator is unsure about when its answer is clear (#71); the vision evaluator is then asked only for the rest. Off: Clef's answer is recorded beside the verdict (clef_score) for comparison." }
+              }
+            }
+          ]
         }
       }
     },
@@ -102,6 +124,14 @@
         "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
       }
     },
+    "pii": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Personal data (#64).",
+      "properties": {
+        "allow": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Values that look like personal data but are not, such as the company's contact e-mail or phone number. * matches any text (*@acme.example). Neither masked nor stopped by kaloko share." }
+      }
+    },
     "share": {
       "type": "object",
       "additionalProperties": false,
@@ -205,6 +235,7 @@
           "description": "Base URL, or per-locale base URLs (key \"default\" is the fallback). ${VAR} is replaced from the environment, e.g. https://${REVIEW_HOST} for review apps."
         },
         "readonly": { "type": "boolean", "default": false },
+        "pii": { "enum": ["required", "redact", "pseudonymize", "off"], "description": "Personal data in runs of this environment. redact or pseudonymize: every run hides it (placeholders, or consistent fake values). required: a run is only created when the scenario or kaloko start --pii hides it, e.g. for production or a local copy of production data." },
         "app": {
           "type": "object",
           "additionalProperties": false,

package/schemas/run.v1.json

@@ -358,6 +358,13 @@
       "type": "object",
       "description": "node|locale → revision, entry, status and author of a draft head."
     },
+    "pii": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "description": "Personal data hidden in this run (#64): { hidden: true, mode: redact | pseudonymize, auto, captures }. The canvas says so, so a blank place is not read as a bug."
+    },
     "app_version": {
       "type": [
         "string",
@@ -497,6 +504,10 @@
                 "quality": {
                   "type": "object",
                   "description": "Capture quality when not the canvas default: format, scale, masked (count), annotations [{n, criterion, x, y}]"
+                },
+                "pii": {
+                  "type": "object",
+                  "description": "Personal data hidden on this capture (#64): mode (redact, pseudonymize), auto (detection on), kinds {kind: count}, rects [{x, y, w, h, kind}] in CSS px of the page, marked on the canvas."
                 }
               }
             }

package/schemas/scenario.v1.json

@@ -186,6 +186,10 @@
     },
     "capture": {
       "$ref": "#/$defs/capture"
+    },
+    "pii": {
+      "enum": ["redact", "pseudonymize"],
+      "description": "Personal data in every capture of the flow: redact hides it behind placeholders and boxes, pseudonymize replaces it with fake values that stay the same for one value throughout the run. Both detect e-mails, phone numbers, birth dates, birth numbers, IBANs, account numbers and addresses; capture.mask adds selectors."
     }
   },
   "$defs": {
@@ -849,11 +853,11 @@
           "description": "WebP quality"
         },
         "mask": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "description": "CSS selectors covered by a solid box on the image (personal data)"
+          "oneOf": [
+            { "type": "array", "items": { "type": "string", "minLength": 1 } },
+            { "const": "auto" }
+          ],
+          "description": "Personal data to hide in every output of the capture (screenshot, HTML, page outline, evaluators, trace, recording): CSS selectors, auto (detect e-mails, phone numbers, birth dates, birth numbers, IBANs, account numbers and addresses), or selectors with auto among them. The scenario's and the step's masks add up."
         },
         "annotate": {
           "type": "boolean",

package/skills/kaloko/SKILL.md

@@ -102,7 +102,9 @@ step or accept a run on a person's behalf, and never hand out a signed
 acceptance record (`kaloko signoff`) for a run a person has not accepted; report what waits for them
 (`kaloko review`, MCP `get_review_queue`) instead. Ask the person
 for secret values (passwords, tokens, gate credentials) and for sign-ins only
-they can do; never write secrets into the config, a scenario or a script.
+they can do; never write secrets into the config, a scenario or a script. When
+`kaloko share` stops on personal data, mask it and capture again; `--allow-pii`
+only when a person says that value may be uploaded.
 
 ## Where to look
 

package/skills/kaloko/reference/acceptance.md

@@ -153,7 +153,7 @@ documentation, manuals and training material set print quality on the scenario o
 capture:
   format: png          # lossless
   scale: 2             # 2× device pixels, crisp in a PDF
-  mask: ["[data-qa=email]", ".customer-name"]   # solid box over personal data (image only; the HTML keeps it)
+  mask: ["[data-qa=email]", ".customer-name"]   # personal data hidden in the image, HTML, outline, trace and video
   annotate: true       # numbered marks on the elements of criteria with `highlight`, in criteria order
 ```
 
@@ -161,7 +161,13 @@ A node's `capture:` overrides the scenario's. Crops for a manual take padding in
 `await crop('#checkout-summary', 'Order summary', { padding: 16 })`. Print quality costs storage (a
 2× PNG is several times a 1× WebP), so keep it for documentation scenarios, not for every run. The
 canvas still loads a small WebP thumbnail; the lightbox and `latest.image` serve the full file.
-Masks hide pixels, not the captured HTML: do not use production personal data in documentation runs.
+Masks cover every output of the step: the screenshot gets a striped box, the HTML and the page outline get
+`[hidden]` of the same length, traces, recordings and captured e-mails are covered too; checks still read the real
+page. On data copied from production add `pii: redact` to the scenario (automatic detection of e-mails, phones,
+birth numbers, IBANs, addresses) or `pii: pseudonymize` (consistent fake values, readable screens). An environment
+with `pii: required` refuses a run without it, and `kaloko share` stops on an unmasked e-mail or phone number and
+names the step and element: mask it and capture again, list a company contact under `pii.allow` in the config, and
+use `--allow-pii` only when a person says the value may go up.
 
 ## Keeping plan and scenario aligned
 

package/skills/kaloko/reference/evaluation.md

@@ -68,6 +68,18 @@ evaluators:
 Report the reason with each `needs_review` or failing criterion; when the
 evaluator saw the screenshot, say so.
 
+## Triage findings (experimental)
+
+When the organization switched the experiment on, `kaloko evaluate` also asks
+Clef, a model on Cloudflare, what kind of problem each screen and a few frames
+of its recording show: overlap, cut-off text, raw keys, placeholders, errors,
+endless loading, layout jumps and more. They land in the eval file and the run
+as `findings` (kind, probability, severity, frame times) and never change a
+verdict. Mention the shown ones (`shown: true`) as "possible problems", apart
+from the verdicts, and fix the clear ones; `get_findings` lists them with the
+answers reviewers gave. Do not answer findings on a reviewer's behalf unless
+asked. `evaluators.triage: false` turns it off.
+
 When a criterion comes back `needs_review`, read the reduced state the
 evaluator saw (`steps/<node>/<locale>/<viewport>.capture.json`, key `state`)
 and the screenshot. Either the page is genuinely unclear (report it), the

package/src/cli.mjs

@@ -62,6 +62,7 @@ import { passesOf, enginesOf, contextOptionsOf, launchEngine, pickKeys, variantF
 import { startScope, scopeKeys, dimensionLine, baseViewports, engineKeys, appDeviceGroups } from './lib/walk-dimensions.mjs'
 import { deviceConfig, prepareAndroid, readyAndroid, finishAndroid, prepareSimulator, applyAppAppearance } from './lib/apps/devices.mjs'
 import { attachAndroidWebView } from './lib/apps/webview.mjs'
+import { piiSettings, piiFindings, guardAllow, formatFindings } from './lib/pii.mjs'
 
 
 const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
@@ -117,6 +118,8 @@ Run
        [--states empty,error]       Only these step states (states: on a step; keys like mobile.state-empty)
        [--token-modes brand-a,brand-b | --token-modes none]
                                     Design environments: the token modes to render (default every mode of the set)
+       [--pii redact|pseudonymize|off]   Hide personal data in every capture (placeholders or consistent fake values);
+                                    default from the environment's or the scenario's pii
   browser start|stop|status [--env e] [--headed]   One shared browser per environment (port = browser.port + index)
                                     Shared Chromium with CDP (agents, scripts and people drive it)
   browser install [--with-deps] [--browsers chromium,webkit,firefox]
@@ -198,6 +201,8 @@ Results
                                     hash, so unchanged captures are not sent again and a broken-off share resumes;
                                     --pr adds or updates one comment on the branch's pull request via gh (KALOKO_PR in
                                     CI), or a note on the GitLab merge request (KALOKO_GITLAB_TOKEN)
+       [--allow-pii]                An unmasked e-mail, phone number, birth number or IBAN in the run stops the upload
+                                    and names the step and element; --allow-pii uploads anyway
   feedback [--run id] [--scenario id] [--json]
                                     Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies;
                                     returned steps first, with their notes
@@ -518,7 +523,7 @@ async function designImport (args) {
   await commands.start({ ...args, _: [], scenario: scenarioFile, env: env.name })
   const current = await resolveRun({ root, output: config.output })
   await commands.walk({ _: [], run: current.dir })
-  await commands.share({ _: [], run: current.dir, ...(typeof args.notes === 'string' ? { notes: args.notes } : {}), ...(typeof args.name === 'string' ? { name: args.name } : {}), ...(typeof args.branch === 'string' ? { branch: args.branch } : {}) })
+  await commands.share({ _: [], run: current.dir, ...(args['allow-pii'] ? { 'allow-pii': true } : {}), ...(typeof args.notes === 'string' ? { notes: args.notes } : {}), ...(typeof args.name === 'string' ? { name: args.name } : {}), ...(typeof args.branch === 'string' ? { branch: args.branch } : {}) })
   return r
 }
 
@@ -606,7 +611,7 @@ async function catalogFor (config, root) {
 const expandRun = (value, short) => (value || '').replace('{run}', short)
 
 // Finds the e-mail (or takes the one given) and captures it as the node's screens.
-async function captureMail ({ config, root, run, env, node, locale, to, subject, timeoutMs = 60000, viewports, message = null, since, id = null }) {
+async function captureMail ({ config, root, run, env, scenario = null, node, locale, to, subject, timeoutMs = 60000, viewports, message = null, since, id = null }) {
   if (!message) {
     const client = mailClient(env.mail, root, env)
     // only messages that arrived during this run unless told otherwise (#52): stale e-mails are not captured silently
@@ -631,8 +636,9 @@ async function captureMail ({ config, root, run, env, node, locale, to, subject,
     // Remote images are allowed for the screenshot but must not hold the capture hostage.
     await page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 20000 })
     await page.waitForLoadState('load', { timeout: 15000 }).catch(() => {})
-    const mail = { id: message.id, subject: message.subject, from: message.from || '', to: (message.to || []).join(', '), date: message.date }
-    const results = await captureStep({ page, node, locale, viewports: scopeViewports(run, viewports), run, maxChars: config.evaluators?.jev?.max_state_chars, extra: { mail } })
+    const mail = { id: message.id, subject: message.subject, from: message.from || '', to: (message.to || []).join(', '), ...(message.cc?.length ? { cc: message.cc.join(', ') } : {}), date: message.date }
+    // the scenario's masks apply to e-mails too (#64); recipients are masked whenever anything is
+    const results = await captureStep({ page, node, locale, viewports: scopeViewports(run, viewports), run, maxChars: config.evaluators?.jev?.max_state_chars, options: captureOptions(scenario, node), extra: { mail } })
     return { subject: mail.subject, count: results.length }
   } finally {
     await browser.close()
@@ -803,7 +809,7 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
             if (node.kind === 'stack') { pending.push(`${node.id} (${locale}): stack steps list web pages; not walked on ${platform}`); continue }
             if (node.kind === 'email' && group !== groups[0]) continue // one message, captured on the first device
             if (node.kind === 'email' && node.mail?.to) {
-              try { const r = await captureMail({ config, root, run, env, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
+              try { const r = await captureMail({ config, root, run, env, scenario, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
               continue
             }
             const scripted = ['playwright', 'script'].includes(node.execution?.strategy) && node.execution?.script
@@ -1077,6 +1083,8 @@ const commands = {
     const env = environment(config, args.env)
     if (scenario.environments && !scenario.environments.includes(env.name)) throw new UserError(`Scenario ${scenario.id} is not allowed on ${env.name} (allowed: ${scenario.environments.join(', ')}).`)
     if (env.readonly && !scenario.readonly) throw new UserError(`Environment ${env.name} is readonly; the scenario must declare readonly: true.`)
+    // personal data (#64): an environment with pii: required gets no run that would store it unmasked
+    const pii = piiSettings({ config, env, scenario, args })
     const locales = list(args.locales) || scenario.locales || ['default']
     // design environments render every mode of their token set (#46): the modes come from the folder's set
     const tokenModes = env.kind === 'design' ? await designTokenModesOf(config, root, env) : []
@@ -1089,8 +1097,9 @@ const commands = {
     const appVer = typeof args['app-version'] === 'string' ? args['app-version'].slice(0, 80) : await appVersion(root)
     // the product the flow documents (visual changelog): its tag, or the only product of the config
     const product = scenario.product || (Object.keys(config.products || {}).length === 1 ? Object.keys(config.products)[0] : null)
-    const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer, product })
+    const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer, product, pii })
     say(`Run ${meta.id}\n  dir: ${relative(process.cwd(), dir)}\n  env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n  locales: ${locales.join(', ')}  viewports: ${vpNames.length > 12 ? `${vpNames.slice(0, 12).join(', ')} … (${vpNames.length})` : vpNames.join(', ')}${dimensionLine(scope)}\n  project: ${project.slug}${issue ? `  issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
+    if (pii) say(`  personal data: ${pii.mode === 'pseudonymize' ? 'replaced with fake values' : 'hidden'} in every capture`)
     const addr = recipient(env.mail, meta.short)
     if (addr) say(`  test e-mail: ${addr}`)
     say(`  other agents may start runs here too: pin this one with --run ${meta.id} or KALOKO_RUN=${meta.id}`)
@@ -1189,7 +1198,7 @@ const commands = {
       const target = args.url === true || (!conn && !args.url) ? urlFor(env, node, locale) : args.url
       if (target) await page.goto(target, { waitUntil: 'domcontentloaded' })
       if (args.crop) {
-        const entry = await captureCrop({ page, node, locale, selector: args.crop, title: args.title, run })
+        const entry = await captureCrop({ page, node, locale, selector: args.crop, title: args.title, run, options: captureOptions(scenario, node) })
         say(`Attached ${entry.file} to ${node.id}`)
         return
       }
@@ -1222,7 +1231,7 @@ const commands = {
     let since
     if (args['any-time']) since = null
     else if (typeof args.since === 'string') { const t = Date.parse(args.since); if (Number.isNaN(t)) throw new UserError(`--since ${args.since} is not a date (ISO 8601, e.g. 2026-09-27T10:00).`); since = new Date(t).toISOString() }
-    const results = await captureMail({ config, root, run, env, node, locale, to, subject, timeoutMs: (Number(args.timeout) || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
+    const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject, timeoutMs: (Number(args.timeout) || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
     say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
   },
 
@@ -1382,7 +1391,7 @@ const commands = {
                 // an e-mail node with a recipient is walked like a step: wait for the message and capture it
                 try {
                   const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
-                  const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
+                  const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
                   say(`✓ ${node.id} ${locale}${tag}: "${results.subject}"`)
                 } catch (error) {
                   failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
@@ -1406,7 +1415,7 @@ const commands = {
                 const records = []
                 const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeKeys(run.meta.scope, stateKeys(run.meta.scope, Object.keys(scopeKeys(run.meta.scope, pass.keys, opts.viewports || null)), state)), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
                 try {
-                  await mod.default({ page: sPage, context: sCtx, locale, state, node: { id: node.id, title: localized(node.title, locale), kind: node.kind, ...(node.states ? { states: nodeStates(run.meta.scope, node) } : {}) }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
+                  await mod.default({ page: sPage, context: sCtx, locale, state, node: { id: node.id, title: localized(node.title, locale), kind: node.kind, ...(node.states ? { states: nodeStates(run.meta.scope, node) } : {}) }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0, options: captureOptions(scenario, node) }), acceptDialogs: () => acceptDialogs(sPage) })
                   if (!captured) await capture()
                 } catch (error) { await rec?.afterStep(node, { error }); throw error }
                 await rec?.afterStep(node, { records })
@@ -1735,6 +1744,7 @@ const commands = {
     const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
     say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
     if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
+    if (report.triage) say(report.triage.screens ? `Triage (experimental, ${report.triage.model}): ${report.triage.screens} screen(s)${report.triage.frames ? ` and ${report.triage.frames} recording frame(s)` : ''}, ${report.triage.shown} finding(s) to look at${report.triage.findings > report.triage.shown ? ` and ${report.triage.findings - report.triage.shown} less likely` : ''}; they never change a verdict.${report.triage.skipped ? ` Stopped early: ${report.triage.skipped}` : ''}` : `Triage (experimental): not run: ${report.triage.skipped}`)
     if (report.skipped.length) {
       // one line per reason: "12 captures: TYPESAFE_API_KEY is not set …"
       const byReason = new Map()
@@ -1800,6 +1810,11 @@ const commands = {
         for (const it of idx.items) for (const byVp of Object.values(it.captures || {})) for (const c of Object.values(byVp)) [c.png, c.thumb, c.html, c.eval].filter(Boolean).forEach((p) => files.add(p))
       }
     }
+    // personal data (#64): nothing goes up with an unmasked e-mail, phone number, birth number or IBAN in it
+    const views = manifest.nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((vps) => Object.values(vps).map((c) => c.view?.revision).filter(Boolean))).map((id) => `live/${id}/index.html`)
+    const findings = await piiFindings({ dir: run.dir, files: [...files, ...views], manifest, allow: guardAllow({ config, env, scenario, run }) })
+    if (findings.length && !args['allow-pii']) throw new UserError(`Not shared: the run holds ${findings.length} unmasked personal data value(s):\n${formatFindings(findings)}\nMask them (capture.mask in the scenario, or pii: redact for automatic detection) and capture the steps again. Values that are not personal data, such as a company contact, go under pii.allow in kaloko.config.yml. To share anyway: kaloko share --allow-pii.`)
+    if (findings.length) say(`Sharing ${findings.length} unmasked personal data value(s), as --allow-pii says:\n${formatFindings(findings, 5)}`)
     const days = Number(args.expires) || config.share?.expires_days || 30
     // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
     let entries = await hashFiles(run.dir, [...files])

package/src/lib/build.mjs

@@ -6,7 +6,7 @@ import { readJson, writeJson } from './util.mjs'
 import { readCaptures, runPaths, listRuns } from './run.mjs'
 import { latestCaptures, evaluationSource, stepIdOf } from './evaluate.mjs'
 import { layout } from './scenario.mjs'
-import { assembleNode, assembleStack, summarize } from './manifest-core.mjs'
+import { assembleNode, assembleStack, summarize, runPii } from './manifest-core.mjs'
 import { packageRoot } from './schema.mjs'
 import { variantOf } from './dimensions.mjs'
 
@@ -82,6 +82,8 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
     nodes,
     edges: scenario.edges.map((e) => ({ from: e.from, to: e.to, label: e.label || '', kind: e.kind || 'main' })),
     summary,
+    // "personal data hidden" (#64): the canvas says so, so a blank is not taken for a bug
+    ...(runPii(meta, nodes) ? { pii: runPii(meta, nodes) } : {}),
     coverage: cov,
     history,
     catalog,

package/src/lib/capture.mjs

@@ -5,6 +5,7 @@ import { probe } from './probe.mjs'
 import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
+import { maskSpec, maskList, inPage, maskInPage, rememberPii, knownPii, maskMailHeaders } from './pii.mjs'
 
 /**
  * Encodes a PNG screenshot inside the page (no native dependency): the image as WebP (or PNG, lossless) and a
@@ -16,7 +17,14 @@ export async function encodeImage (page, png, { format = 'webp', quality = 0.82,
     await new Promise((res, rej) => { img.onload = res; img.onerror = rej; img.src = dataUrl })
     const full = document.createElement('canvas'); full.width = img.width; full.height = img.height
     const g = full.getContext('2d'); g.drawImage(img, 0, 0)
-    g.fillStyle = '#1f1f1d'; for (const r of masks) g.fillRect(Math.floor(r.x * scale), Math.floor(r.y * scale), Math.ceil(r.w * scale), Math.ceil(r.h * scale))
+    // masked personal data: a dark box with diagonal stripes, so a reader sees "hidden" and not a broken page (#64)
+    for (const r of masks) {
+      const x = Math.floor(r.x * scale); const y = Math.floor(r.y * scale); const w = Math.ceil(r.w * scale); const h = Math.ceil(r.h * scale)
+      g.fillStyle = '#1f1f1d'; g.fillRect(x, y, w, h)
+      g.save(); g.beginPath(); g.rect(x, y, w, h); g.clip(); g.strokeStyle = '#3d3d38'; g.lineWidth = 3 * scale
+      for (let i = -h; i < w; i += 12 * scale) { g.beginPath(); g.moveTo(x + i, y + h); g.lineTo(x + i + h, y); g.stroke() }
+      g.restore()
+    }
     for (const m of marks) {
       const r = 13 * scale; const x = Math.max(r + 2, m.x * scale); const y = Math.max(r + 2, m.y * scale)
       g.beginPath(); g.arc(x, y, r, 0, Math.PI * 2); g.fillStyle = '#ffe452'; g.fill(); g.lineWidth = 2 * scale; g.strokeStyle = '#151514'; g.stroke()
@@ -50,12 +58,15 @@ export async function stepSpec (page, tokens = null, mode = null) {
 
 export function captureOptions (scenario, node) {
   const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
+  // masks add up: a step's own selectors never drop the scenario's (personal data stays hidden on every step)
+  const masks = [maskList(scenario?.capture?.mask), maskList(node?.capture?.mask)]
   const q = Number(o.quality)
   return {
     format: o.format === 'png' ? 'png' : 'webp',
     scale: Number(o.scale) === 2 ? 2 : 1,
     quality: Number.isFinite(q) && q > 0 ? Math.min(1, Math.max(0.3, q)) : 0.82,
-    mask: Array.isArray(o.mask) ? o.mask.map(String).filter(Boolean) : [],
+    mask: [...new Set(masks.flatMap((m) => m.selectors))],
+    maskAuto: masks.some((m) => m.auto),
     annotate: o.annotate === true,
     // stack pages default to the first screen: hundreds of full-page shots would swamp storage
     fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack'
@@ -121,6 +132,9 @@ export async function captureStep ({ page, node, locale, viewports, run, console
   const original = page.viewportSize()
   const results = []
   let netResults = null
+  // personal data (#64): what this capture masks; recipients of a captured e-mail are masked before the body
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  if (mask.active && extra.mail) extra = { ...extra, mail: maskMailHeaders(extra.mail, mask) }
   // batch E: the engine of this page, and per key its colour scheme, motion and display mode (run scope variants)
   const engine = engineOf(page)
   const scope = run.meta?.scope
@@ -128,14 +142,38 @@ export async function captureStep ({ page, node, locale, viewports, run, console
     const now = page.viewportSize()
     if (!now || now.width !== width || now.height !== height) await page.setViewportSize({ width, height })
     const undoVariant = await applyVariant(page, scope?.variants?.[viewport], scope?.devices?.[scope?.variants?.[viewport]?.device] || null)
+    let masking = null
     try {
       await settle(page)
       const html = paths.step(stepId, locale, viewport, 'html')
       await rm(paths.step(stepId, locale, viewport, 'eval.json'), { force: true })
       await mkdir(dirname(html), { recursive: true })
-      // PII selectors are covered by a solid box; at 2× Chrome renders the full page at double scale (CDP), and
-      // the boxes are drawn while encoding because Playwright's own mask only works at the page's pixel ratio
-      const maskRects = opts.mask.length ? await page.evaluate((sels) => sels.flatMap((sel) => { try { return [...document.querySelectorAll(sel)] } catch { return [] } }).map((el) => { const r = el.getBoundingClientRect(); return { x: r.left + scrollX, y: r.top + scrollY, w: r.width, h: r.height } }).filter((r) => r.w > 0 && r.h > 0), opts.mask) : []
+      // the checks read the page as it is: masks change what is stored, never a verdict
+      let result
+      for (let attempt = 0; ; attempt++) {
+        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
+          // the page navigated under us (redirect, Turbo); wait once and probe the new document
+          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
+          throw error
+        }
+      }
+      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
+      const ignore = await resolveIgnore(page, node, locale, viewport)
+      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the mask and the keyboard walk
+      await liveValueChecks({ page, node, result, url: page.url(), tokens })
+      // One mask for every output (#64): from here to the end of the capture the page shows placeholders or fake
+      // values instead of personal data, so the screenshot, HTML, outline, spec, keyboard shots and HTML view agree
+      if (mask.active) {
+        masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
+        rememberPii(masking.pairs)
+        await page.evaluate((known) => window.__kalokoMask?.update({ known }), knownPii()).catch(() => {})
+        // the outline the evaluators read comes from the masked page; checks and boxes keep the first probe
+        if (masking.rects.length || masking.pairs.length || Object.keys(masking.counts).length) {
+          const again = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)).catch(() => null)
+          if (again) for (const k of ['state', 'scoped', 'overflow', 'meta', 'design']) if (again[k] !== undefined) result[k] = again[k]
+        }
+      }
+      const maskRects = (masking?.rects || []).filter((r) => r.box)
       let shot
       // a device preset's pixel ratio (3 on an iPhone) shapes the layout, not the image: captures stay in CSS pixels
       // (or 2× for documentation), so the canvas, boxes and diffs work the same for every device
@@ -152,22 +190,12 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         // WebKit and Firefox have no 2× capture of their own: 2× when the context renders at 2× or more, else 1×
         // (exactly 2×: a 3× or 2.625× device would give an image the masks and marks are not drawn for)
         if (opts.scale === 2 && dpr !== 2) shotScale = 1
-        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css', ...(opts.mask.length && shotScale === 1 ? { mask: opts.mask.map((sel) => page.locator(sel)), maskColor: '#1f1f1d' } : {}) })
+        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css' })
       }
       // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
       await writeFile(html, redact(await page.content()))
-      let result
-      for (let attempt = 0; ; attempt++) {
-        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
-          // the page navigated under us (redirect, Turbo); wait once and probe the new document
-          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
-          throw error
-        }
-      }
       // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
       if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens, scope?.variants?.[viewport]?.token_mode || scope?.variants?.[viewport]?.color_scheme || null)
-      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
-      const ignore = await resolveIgnore(page, node, locale, viewport)
       // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
       const unstable = stability > 1 ? await repeatRegions(page, { n: Math.min(6, stability), fullPage: opts.fullPage }).catch((error) => ({ error: String(error.message).slice(0, 200) })) : null
       // numbered marks on the highlighted elements, in criteria order (first box of each)
@@ -176,7 +204,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       let thumb = paths.step(stepId, locale, viewport, 'thumb.webp')
       let thumbWritten = true
       try {
-        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: shotScale === 2 ? maskRects : [] })
+        // mask boxes are drawn while encoding, at the image's own scale (Playwright's mask only works at the page's pixel ratio)
+        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: maskRects })
         // the extension follows the bytes (WebKit encodes PNG), so the service accepts the file
         if (enc.imageType === 'image/png') image = paths.step(stepId, locale, viewport, 'png')
         if (enc.thumbType === 'image/png') thumb = paths.step(stepId, locale, viewport, 'thumb.png')
@@ -184,13 +213,11 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         await writeFile(thumb, enc.thumb)
       } catch (error) {
         // never store an unmasked image when masks were asked for
-        if (shotScale === 2 && maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
+        if (maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
         image = paths.step(stepId, locale, viewport, 'png'); thumbWritten = false
         await writeFile(image, shot)
       }
       const url = page.url()
-      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the keyboard walk moves focus
-      await liveValueChecks({ page, node, result, url, tokens })
       const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
       const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
       // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
@@ -236,7 +263,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       let view = null
       if (htmlView) {
         try {
-          // the same PII masks as the screenshot (#27), applied to the text of the copy
+          // the page is still masked here (#64); the selectors are covered in the copy as well (#27)
           const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])] })
           await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
           view = { revision: rev.id, entry: rev.entry }
@@ -254,6 +281,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...extra,
         http_status: extra.http_status ?? result.http_status ?? null,
         capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+        // what was hidden and where (#64): the canvas marks these places, so a blank is not read as a bug
+        ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: masking?.counts || {}, rects: (masking?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
         ...(ignore.length ? { ignore } : {}),
         ...(unstable ? { unstable } : {}),
         ...(keyboard ? { keyboard } : {}),
@@ -265,27 +294,42 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       await writeJson(file, redactDeep(record))
       await addCapture(run.dir, { type: 'capture', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, file: relative(run.dir, file), at: record.captured_at })
       results.push(record)
-    } finally { await undoVariant() }
+    } finally {
+      // the page gets its own text back, also when the capture failed half-way
+      if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {})
+      await undoVariant()
+    }
   }
   const last = page.viewportSize()
   if (original && (!last || last.width !== original.width || last.height !== original.height)) await page.setViewportSize(original)
   return results
 }
 
-export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0 }) {
+export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0, options = null }) {
   const paths = runPaths(run.dir)
   const el = page.locator(selector).first()
   const name = `${node.id}-${locale}-${Date.now().toString(36)}.png`
   const file = paths.attachment(name)
   await mkdir(dirname(file), { recursive: true })
-  if (padding > 0) {
-    // element plus a margin of surrounding page, clipped from a full-page shot (documentation crops)
+  // the same mask as the step's capture (#64): masked text in the page, boxes over masked elements
+  const opts = options || captureOptions(null, node)
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  let masking = null
+  try {
+    if (mask.active) { masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() })); rememberPii(masking.pairs) }
+    const boxes = (masking?.rects || []).filter((r) => r.box)
+    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
     const box = await el.boundingBox()
     if (!box) throw new Error(`crop: ${selector} is not visible`)
-    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
+    // element plus a margin of surrounding page (documentation crops), clipped from a full-page shot
     const x = Math.max(0, box.x + doc.x - padding); const y = Math.max(0, box.y + doc.y - padding)
-    await page.screenshot({ path: file, fullPage: true, clip: { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) } })
-  } else await el.screenshot({ path: file })
+    const clip = { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) }
+    const png = padding > 0 || boxes.length ? await page.screenshot({ fullPage: true, clip }) : await el.screenshot()
+    if (boxes.length) {
+      const enc = await encodeImage(page, png, { format: 'png', masks: boxes.map((r) => ({ ...r, x: r.x - clip.x, y: r.y - clip.y })) })
+      await writeFile(file, enc.image)
+    } else await writeFile(file, png)
+  } finally { if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
   const entry = { type: 'attachment', kind: 'crop', node: node.id, locale, title: title || selector, selector, file: relative(run.dir, file), viewport: page.viewportSize(), at: new Date().toISOString() }
   await addCapture(run.dir, entry)
   return entry