kaloko 0.11.0 → 0.12.1

package/README.md

@@ -219,6 +219,10 @@ Every AI verdict comes with one sentence saying why. Criteria marked
 `vision: true`, and answers the text evaluator is unsure about, get a second
 look at the screenshot from a vision model (`ANTHROPIC_API_KEY`, or through
 Kaloko); `kaloko build` lists what to act on with the score and the reason.
+As an experiment you can switch on in Settings, Clef on Cloudflare also names
+the kind of problem each screen and its recording show (overlap, cut-off text,
+a raw translation key, an endless spinner); reviewers turn the right findings
+into comments with one click, and findings never change a verdict.
 The `a11y` pack walks the page with Tab (order, visible focus, traps, skip
 link) and reads names from the accessibility tree. Pixel diffs skip the
 regions a step ignores; `kaloko walk --stability 3` and `kaloko stability`
@@ -410,6 +414,18 @@ captured HTML, page text and console output.
 `npx kaloko doctor` checks that every variable the config names is set and that
 each environment lets you in.
 
+**Personal data.** `capture.mask` (CSS selectors) hides personal data in
+everything a step stores: a striped box on the screenshot, `[hidden]` of the
+same length in the HTML and the page outline, and the same places in traces,
+recordings and captured e-mails. `pii: redact` on a scenario finds e-mails,
+phone numbers, birth numbers, IBANs and addresses by itself; `pii:
+pseudonymize` puts consistent fake values in their place instead. An
+environment with `pii: required` gets no run without it. When such a run still
+holds an unmasked e-mail or phone number, `npx kaloko share` refuses it and
+names the step and element (`--allow-pii` overrides); other runs are shared
+with a warning. The site's own contacts (its domain, the footer, Organization
+JSON-LD) do not count. Checks always read the real page.
+
 Set `browser.channel: chrome` to use an installed Chrome, or run
 `npx kaloko browser install` for the Chromium of kaloko's Playwright
 (`KALOKO_BROWSER_CHANNEL=chromium` then uses it, which is how CI runs).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "See what your AI agents built: every screen, language and device walked, assessed and approved on one canvas.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -78,6 +78,29 @@
               }
             }
           ]
+        },
+        "triage": {
+          "description": "Triage (#65, experimental): Cloudflare's Clef names the kind of problem on each evaluated screen and in chosen frames of its recording (overlap, cut-off text, raw key, endless spinner…). Findings are advisory and never change a verdict. Runs through the organization's service once an admin switches the experiment on (Settings → Security → Experiments), or with CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID. false turns it off.",
+          "oneOf": [
+            { "const": false },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "model": { "enum": ["clef", "clef-flash"], "default": "clef", "description": "clef found more on the first bench; clef-flash answers about twice as fast." },
+                "keep": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.7, "description": "Findings at or above this probability show on the canvas; lower ones fold under Less likely." },
+                "maybe": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.5, "description": "Findings below this probability are dropped." },
+                "max_width": { "type": "integer", "minimum": 320, "maximum": 2048, "default": 896 },
+                "tile_height": { "type": "integer", "minimum": 400, "maximum": 4000, "default": 1024, "description": "Clef reads about one token per 40 pixels within 65,536 tokens: keep max_tiles × max_width × tile_height near 1.8 million." },
+                "max_tiles": { "type": "integer", "minimum": 1, "maximum": 4, "default": 2 },
+                "frame_width": { "type": "integer", "minimum": 240, "maximum": 1280, "default": 640, "description": "Width of recording frames sent to Clef (four per request)." },
+                "frames_per_step": { "type": "integer", "minimum": 0, "maximum": 12, "default": 6, "description": "Frames chosen from a step's recording (around taps, sharp changes and the end); 0 skips recordings." },
+                "frames_per_run": { "type": "integer", "minimum": 0, "default": 60 },
+                "max_calls": { "type": "integer", "minimum": 0, "default": 200, "description": "Uncached triage requests per evaluation." },
+                "cascade": { "type": "boolean", "default": false, "description": "Let Clef settle semantic criteria the text evaluator is unsure about when its answer is clear (#71); the vision evaluator is then asked only for the rest. Off: Clef's answer is recorded beside the verdict (clef_score) for comparison." }
+              }
+            }
+          ]
         }
       }
     },
@@ -102,6 +125,15 @@
         "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
       }
     },
+    "pii": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Personal data (#64).",
+      "properties": {
+        "allow": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Values that look like personal data but are not, such as the company's contact e-mail or phone number. * matches any text (*@acme.example). Neither masked nor stopped by kaloko share." },
+        "site_contacts": { "type": "boolean", "default": true, "description": "kaloko share takes the site's own contacts for not personal data: e-mails on the environment's domain, e-mails and phone numbers in the page footer, and those of an Organization or ContactPoint in JSON-LD. false checks them like any other value." }
+      }
+    },
     "share": {
       "type": "object",
       "additionalProperties": false,
@@ -205,6 +237,7 @@
           "description": "Base URL, or per-locale base URLs (key \"default\" is the fallback). ${VAR} is replaced from the environment, e.g. https://${REVIEW_HOST} for review apps."
         },
         "readonly": { "type": "boolean", "default": false },
+        "pii": { "enum": ["required", "redact", "pseudonymize", "off"], "description": "Personal data in runs of this environment. redact or pseudonymize: every run hides it (placeholders, or consistent fake values). required: a run is only created when the scenario or kaloko start --pii hides it, e.g. for production or a local copy of production data." },
         "app": {
           "type": "object",
           "additionalProperties": false,

package/schemas/run.v1.json

@@ -358,6 +358,13 @@
       "type": "object",
       "description": "node|locale → revision, entry, status and author of a draft head."
     },
+    "pii": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "description": "Personal data hidden in this run (#64): { hidden: true, mode: redact | pseudonymize, auto, captures }. The canvas says so, so a blank place is not read as a bug."
+    },
     "app_version": {
       "type": [
         "string",
@@ -497,6 +504,10 @@
                 "quality": {
                   "type": "object",
                   "description": "Capture quality when not the canvas default: format, scale, masked (count), annotations [{n, criterion, x, y}]"
+                },
+                "pii": {
+                  "type": "object",
+                  "description": "Personal data hidden on this capture (#64): mode (redact, pseudonymize), auto (detection on), kinds {kind: count}, rects [{x, y, w, h, kind}] in CSS px of the page, marked on the canvas."
                 }
               }
             }

package/schemas/scenario.v1.json

@@ -186,6 +186,10 @@
     },
     "capture": {
       "$ref": "#/$defs/capture"
+    },
+    "pii": {
+      "enum": ["redact", "pseudonymize"],
+      "description": "Personal data in every capture of the flow: redact hides it behind placeholders and boxes, pseudonymize replaces it with fake values that stay the same for one value throughout the run. Both detect e-mails, phone numbers, birth dates, birth numbers, IBANs, account numbers and addresses; capture.mask adds selectors."
     }
   },
   "$defs": {
@@ -849,11 +853,11 @@
           "description": "WebP quality"
         },
         "mask": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "description": "CSS selectors covered by a solid box on the image (personal data)"
+          "oneOf": [
+            { "type": "array", "items": { "type": "string", "minLength": 1 } },
+            { "const": "auto" }
+          ],
+          "description": "Personal data to hide in every output of the capture (screenshot, HTML, page outline, evaluators, trace, recording): CSS selectors, auto (detect e-mails, phone numbers, birth dates, birth numbers, IBANs, account numbers and addresses), or selectors with auto among them. The scenario's and the step's masks add up."
         },
         "annotate": {
           "type": "boolean",

package/skills/kaloko/SKILL.md

@@ -102,7 +102,9 @@ step or accept a run on a person's behalf, and never hand out a signed
 acceptance record (`kaloko signoff`) for a run a person has not accepted; report what waits for them
 (`kaloko review`, MCP `get_review_queue`) instead. Ask the person
 for secret values (passwords, tokens, gate credentials) and for sign-ins only
-they can do; never write secrets into the config, a scenario or a script.
+they can do; never write secrets into the config, a scenario or a script. When
+`kaloko share` stops on personal data, mask it and capture again; `--allow-pii`
+only when a person says that value may be uploaded.
 
 ## Where to look
 

package/skills/kaloko/reference/acceptance.md

@@ -153,7 +153,7 @@ documentation, manuals and training material set print quality on the scenario o
 capture:
   format: png          # lossless
   scale: 2             # 2× device pixels, crisp in a PDF
-  mask: ["[data-qa=email]", ".customer-name"]   # solid box over personal data (image only; the HTML keeps it)
+  mask: ["[data-qa=email]", ".customer-name"]   # personal data hidden in the image, HTML, outline, trace and video
   annotate: true       # numbered marks on the elements of criteria with `highlight`, in criteria order
 ```
 
@@ -161,7 +161,13 @@ A node's `capture:` overrides the scenario's. Crops for a manual take padding in
 `await crop('#checkout-summary', 'Order summary', { padding: 16 })`. Print quality costs storage (a
 2× PNG is several times a 1× WebP), so keep it for documentation scenarios, not for every run. The
 canvas still loads a small WebP thumbnail; the lightbox and `latest.image` serve the full file.
-Masks hide pixels, not the captured HTML: do not use production personal data in documentation runs.
+Masks cover every output of the step: the screenshot gets a striped box, the HTML and the page outline get
+`[hidden]` of the same length, traces, recordings and captured e-mails are covered too; checks still read the real
+page. On data copied from production add `pii: redact` to the scenario (automatic detection of e-mails, phones,
+birth numbers, IBANs, addresses) or `pii: pseudonymize` (consistent fake values, readable screens). An environment
+with `pii: required` refuses a run without it, and in a run that hides personal data `kaloko share` stops on an
+unmasked e-mail or phone number and names the step and element (other runs are shared with a warning): mask it and capture again, list a company contact under `pii.allow` in the config, and
+use `--allow-pii` only when a person says the value may go up.
 
 ## Keeping plan and scenario aligned
 

package/skills/kaloko/reference/evaluation.md

@@ -68,6 +68,18 @@ evaluators:
 Report the reason with each `needs_review` or failing criterion; when the
 evaluator saw the screenshot, say so.
 
+## Triage findings (experimental)
+
+When the organization switched the experiment on, `kaloko evaluate` also asks
+Clef, a model on Cloudflare, what kind of problem each screen and a few frames
+of its recording show: overlap, cut-off text, raw keys, placeholders, errors,
+endless loading, layout jumps and more. They land in the eval file and the run
+as `findings` (kind, probability, severity, frame times) and never change a
+verdict. Mention the shown ones (`shown: true`) as "possible problems", apart
+from the verdicts, and fix the clear ones; `get_findings` lists them with the
+answers reviewers gave. Do not answer findings on a reviewer's behalf unless
+asked. `evaluators.triage: false` turns it off.
+
 When a criterion comes back `needs_review`, read the reduced state the
 evaluator saw (`steps/<node>/<locale>/<viewport>.capture.json`, key `state`)
 and the screenshot. Either the page is genuinely unclear (report it), the

package/src/cli.mjs

@@ -62,6 +62,7 @@ import { passesOf, enginesOf, contextOptionsOf, launchEngine, pickKeys, variantF
 import { startScope, scopeKeys, dimensionLine, baseViewports, engineKeys, appDeviceGroups } from './lib/walk-dimensions.mjs'
 import { deviceConfig, prepareAndroid, readyAndroid, finishAndroid, prepareSimulator, applyAppAppearance } from './lib/apps/devices.mjs'
 import { attachAndroidWebView } from './lib/apps/webview.mjs'
+import { piiSettings, piiFindings, piiRequested, guardAllow, formatFindings } from './lib/pii.mjs'
 
 
 const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
@@ -117,6 +118,8 @@ Run
        [--states empty,error]       Only these step states (states: on a step; keys like mobile.state-empty)
        [--token-modes brand-a,brand-b | --token-modes none]
                                     Design environments: the token modes to render (default every mode of the set)
+       [--pii redact|pseudonymize|off]   Hide personal data in every capture (placeholders or consistent fake values);
+                                    default from the environment's or the scenario's pii
   browser start|stop|status [--env e] [--headed]   One shared browser per environment (port = browser.port + index)
                                     Shared Chromium with CDP (agents, scripts and people drive it)
   browser install [--with-deps] [--browsers chromium,webkit,firefox]
@@ -198,6 +201,9 @@ Results
                                     hash, so unchanged captures are not sent again and a broken-off share resumes;
                                     --pr adds or updates one comment on the branch's pull request via gh (KALOKO_PR in
                                     CI), or a note on the GitLab merge request (KALOKO_GITLAB_TOKEN)
+       [--allow-pii]                In a run that hides personal data, an unmasked e-mail, phone number, birth number or
+                                    IBAN stops the upload and names the step and element; --allow-pii uploads anyway
+                                    (other runs upload with a warning)
   feedback [--run id] [--scenario id] [--json]
                                     Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies;
                                     returned steps first, with their notes
@@ -518,7 +524,7 @@ async function designImport (args) {
   await commands.start({ ...args, _: [], scenario: scenarioFile, env: env.name })
   const current = await resolveRun({ root, output: config.output })
   await commands.walk({ _: [], run: current.dir })
-  await commands.share({ _: [], run: current.dir, ...(typeof args.notes === 'string' ? { notes: args.notes } : {}), ...(typeof args.name === 'string' ? { name: args.name } : {}), ...(typeof args.branch === 'string' ? { branch: args.branch } : {}) })
+  await commands.share({ _: [], run: current.dir, ...(args['allow-pii'] ? { 'allow-pii': true } : {}), ...(typeof args.notes === 'string' ? { notes: args.notes } : {}), ...(typeof args.name === 'string' ? { name: args.name } : {}), ...(typeof args.branch === 'string' ? { branch: args.branch } : {}) })
   return r
 }
 
@@ -606,7 +612,7 @@ async function catalogFor (config, root) {
 const expandRun = (value, short) => (value || '').replace('{run}', short)
 
 // Finds the e-mail (or takes the one given) and captures it as the node's screens.
-async function captureMail ({ config, root, run, env, node, locale, to, subject, timeoutMs = 60000, viewports, message = null, since, id = null }) {
+async function captureMail ({ config, root, run, env, scenario = null, node, locale, to, subject, timeoutMs = 60000, viewports, message = null, since, id = null }) {
   if (!message) {
     const client = mailClient(env.mail, root, env)
     // only messages that arrived during this run unless told otherwise (#52): stale e-mails are not captured silently
@@ -631,8 +637,9 @@ async function captureMail ({ config, root, run, env, node, locale, to, subject,
     // Remote images are allowed for the screenshot but must not hold the capture hostage.
     await page.setContent(html, { waitUntil: 'domcontentloaded', timeout: 20000 })
     await page.waitForLoadState('load', { timeout: 15000 }).catch(() => {})
-    const mail = { id: message.id, subject: message.subject, from: message.from || '', to: (message.to || []).join(', '), date: message.date }
-    const results = await captureStep({ page, node, locale, viewports: scopeViewports(run, viewports), run, maxChars: config.evaluators?.jev?.max_state_chars, extra: { mail } })
+    const mail = { id: message.id, subject: message.subject, from: message.from || '', to: (message.to || []).join(', '), ...(message.cc?.length ? { cc: message.cc.join(', ') } : {}), date: message.date }
+    // the scenario's masks apply to e-mails too (#64); recipients are masked whenever anything is
+    const results = await captureStep({ page, node, locale, viewports: scopeViewports(run, viewports), run, maxChars: config.evaluators?.jev?.max_state_chars, options: captureOptions(scenario, node), extra: { mail } })
     return { subject: mail.subject, count: results.length }
   } finally {
     await browser.close()
@@ -803,7 +810,7 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
             if (node.kind === 'stack') { pending.push(`${node.id} (${locale}): stack steps list web pages; not walked on ${platform}`); continue }
             if (node.kind === 'email' && group !== groups[0]) continue // one message, captured on the first device
             if (node.kind === 'email' && node.mail?.to) {
-              try { const r = await captureMail({ config, root, run, env, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
+              try { const r = await captureMail({ config, root, run, env, scenario, node, locale, to: expandRun(node.mail.to, run.meta.short), subject: localized(node.mail?.subject, locale), timeoutMs: 45000 }); say(`✓ ${node.id} ${locale}: "${r.subject}"`) } catch (error) { failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`) }
               continue
             }
             const scripted = ['playwright', 'script'].includes(node.execution?.strategy) && node.execution?.script
@@ -1077,6 +1084,8 @@ const commands = {
     const env = environment(config, args.env)
     if (scenario.environments && !scenario.environments.includes(env.name)) throw new UserError(`Scenario ${scenario.id} is not allowed on ${env.name} (allowed: ${scenario.environments.join(', ')}).`)
     if (env.readonly && !scenario.readonly) throw new UserError(`Environment ${env.name} is readonly; the scenario must declare readonly: true.`)
+    // personal data (#64): an environment with pii: required gets no run that would store it unmasked
+    const pii = piiSettings({ config, env, scenario, args })
     const locales = list(args.locales) || scenario.locales || ['default']
     // design environments render every mode of their token set (#46): the modes come from the folder's set
     const tokenModes = env.kind === 'design' ? await designTokenModesOf(config, root, env) : []
@@ -1089,8 +1098,9 @@ const commands = {
     const appVer = typeof args['app-version'] === 'string' ? args['app-version'].slice(0, 80) : await appVersion(root)
     // the product the flow documents (visual changelog): its tag, or the only product of the config
     const product = scenario.product || (Object.keys(config.products || {}).length === 1 ? Object.keys(config.products)[0] : null)
-    const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer, product })
+    const { dir, meta } = await createRun({ output: config.output, scenario, scenarioFile, env, scope, git, executor, root, project, issue, appVersion: appVer, product, pii })
     say(`Run ${meta.id}\n  dir: ${relative(process.cwd(), dir)}\n  env: ${env.name}${env.readonly ? ' (readonly)' : ''}\n  locales: ${locales.join(', ')}  viewports: ${vpNames.length > 12 ? `${vpNames.slice(0, 12).join(', ')} … (${vpNames.length})` : vpNames.join(', ')}${dimensionLine(scope)}\n  project: ${project.slug}${issue ? `  issue: ${issue.repo ? issue.repo + '#' : '#'}${issue.number}` : ''}`)
+    if (pii) say(`  personal data: ${pii.mode === 'pseudonymize' ? 'replaced with fake values' : 'hidden'} in every capture`)
     const addr = recipient(env.mail, meta.short)
     if (addr) say(`  test e-mail: ${addr}`)
     say(`  other agents may start runs here too: pin this one with --run ${meta.id} or KALOKO_RUN=${meta.id}`)
@@ -1189,7 +1199,7 @@ const commands = {
       const target = args.url === true || (!conn && !args.url) ? urlFor(env, node, locale) : args.url
       if (target) await page.goto(target, { waitUntil: 'domcontentloaded' })
       if (args.crop) {
-        const entry = await captureCrop({ page, node, locale, selector: args.crop, title: args.title, run })
+        const entry = await captureCrop({ page, node, locale, selector: args.crop, title: args.title, run, options: captureOptions(scenario, node) })
         say(`Attached ${entry.file} to ${node.id}`)
         return
       }
@@ -1222,7 +1232,7 @@ const commands = {
     let since
     if (args['any-time']) since = null
     else if (typeof args.since === 'string') { const t = Date.parse(args.since); if (Number.isNaN(t)) throw new UserError(`--since ${args.since} is not a date (ISO 8601, e.g. 2026-09-27T10:00).`); since = new Date(t).toISOString() }
-    const results = await captureMail({ config, root, run, env, node, locale, to, subject, timeoutMs: (Number(args.timeout) || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
+    const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject, timeoutMs: (Number(args.timeout) || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
     say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
   },
 
@@ -1382,7 +1392,7 @@ const commands = {
                 // an e-mail node with a recipient is walked like a step: wait for the message and capture it
                 try {
                   const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
-                  const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
+                  const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
                   say(`✓ ${node.id} ${locale}${tag}: "${results.subject}"`)
                 } catch (error) {
                   failed.push(`${node.id} (${locale}${tag}): ${error.message.split('\n')[0]}`)
@@ -1406,7 +1416,7 @@ const commands = {
                 const records = []
                 const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeKeys(run.meta.scope, stateKeys(run.meta.scope, Object.keys(scopeKeys(run.meta.scope, pass.keys, opts.viewports || null)), state)), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
                 try {
-                  await mod.default({ page: sPage, context: sCtx, locale, state, node: { id: node.id, title: localized(node.title, locale), kind: node.kind, ...(node.states ? { states: nodeStates(run.meta.scope, node) } : {}) }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
+                  await mod.default({ page: sPage, context: sCtx, locale, state, node: { id: node.id, title: localized(node.title, locale), kind: node.kind, ...(node.states ? { states: nodeStates(run.meta.scope, node) } : {}) }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0, options: captureOptions(scenario, node) }), acceptDialogs: () => acceptDialogs(sPage) })
                   if (!captured) await capture()
                 } catch (error) { await rec?.afterStep(node, { error }); throw error }
                 await rec?.afterStep(node, { records })
@@ -1735,6 +1745,7 @@ const commands = {
     const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
     say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
     if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
+    if (report.triage) say(report.triage.screens ? `Triage (experimental, ${report.triage.model}): ${report.triage.screens} screen(s)${report.triage.frames ? ` and ${report.triage.frames} recording frame(s)` : ''}, ${report.triage.shown} finding(s) to look at${report.triage.findings > report.triage.shown ? ` and ${report.triage.findings - report.triage.shown} less likely` : ''}; they never change a verdict.${report.triage.skipped ? ` Not every request was answered: ${report.triage.skipped}` : ''}` : `Triage (experimental): not run${report.triage.skipped ? `: ${report.triage.skipped}` : ' (no screenshot to look at)'}`)
     if (report.skipped.length) {
       // one line per reason: "12 captures: TYPESAFE_API_KEY is not set …"
       const byReason = new Map()
@@ -1800,6 +1811,14 @@ const commands = {
         for (const it of idx.items) for (const byVp of Object.values(it.captures || {})) for (const c of Object.values(byVp)) [c.png, c.thumb, c.html, c.eval].filter(Boolean).forEach((p) => files.add(p))
       }
     }
+    // personal data (#64): nothing goes up with an unmasked e-mail, phone number, birth number or IBAN in it
+    const views = manifest.nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((vps) => Object.values(vps).map((c) => c.view?.revision).filter(Boolean))).map((id) => `live/${id}/index.html`)
+    // a run that asked for hiding personal data is refused; any other run is shared with a warning
+    const findings = await piiFindings({ dir: run.dir, files: [...files, ...views], manifest, allow: guardAllow({ config, env, scenario, run }), site: { url: run.meta.environment?.base_url || env.base_url, contacts: config.pii?.site_contacts !== false } })
+    const strict = piiRequested({ run, env, scenario })
+    if (findings.length && strict && !args['allow-pii']) throw new UserError(`Not shared: the run holds ${findings.length} unmasked personal data value(s):\n${formatFindings(findings)}\nMask them (capture.mask in the scenario, or pii: redact for automatic detection) and capture the steps again. Values that are not personal data, such as a company contact, go under pii.allow in kaloko.config.yml. To share anyway: kaloko share --allow-pii.`)
+    if (findings.length && strict) say(`Sharing ${findings.length} unmasked personal data value(s), as --allow-pii says:\n${formatFindings(findings, 5)}`)
+    else if (findings.length) say(`This run shows ${findings.length} value(s) that look like personal data:\n${formatFindings(findings, 5)}\nShared anyway, because the run does not hide personal data. To hide it, add pii: redact to the scenario or the environment; values that are not personal data go under pii.allow in kaloko.config.yml.`)
     const days = Number(args.expires) || config.share?.expires_days || 30
     // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
     let entries = await hashFiles(run.dir, [...files])

package/src/lib/build.mjs

@@ -6,7 +6,7 @@ import { readJson, writeJson } from './util.mjs'
 import { readCaptures, runPaths, listRuns } from './run.mjs'
 import { latestCaptures, evaluationSource, stepIdOf } from './evaluate.mjs'
 import { layout } from './scenario.mjs'
-import { assembleNode, assembleStack, summarize } from './manifest-core.mjs'
+import { assembleNode, assembleStack, summarize, runPii } from './manifest-core.mjs'
 import { packageRoot } from './schema.mjs'
 import { variantOf } from './dimensions.mjs'
 
@@ -82,6 +82,8 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
     nodes,
     edges: scenario.edges.map((e) => ({ from: e.from, to: e.to, label: e.label || '', kind: e.kind || 'main' })),
     summary,
+    // "personal data hidden" (#64): the canvas says so, so a blank is not taken for a bug
+    ...(runPii(meta, nodes) ? { pii: runPii(meta, nodes) } : {}),
     coverage: cov,
     history,
     catalog,