kaloko 0.10.0 → 0.12.0

package/src/lib/capture.mjs

@@ -5,6 +5,7 @@ import { probe } from './probe.mjs'
 import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
+import { maskSpec, maskList, inPage, maskInPage, rememberPii, knownPii, maskMailHeaders } from './pii.mjs'
 
 /**
  * Encodes a PNG screenshot inside the page (no native dependency): the image as WebP (or PNG, lossless) and a
@@ -16,7 +17,14 @@ export async function encodeImage (page, png, { format = 'webp', quality = 0.82,
     await new Promise((res, rej) => { img.onload = res; img.onerror = rej; img.src = dataUrl })
     const full = document.createElement('canvas'); full.width = img.width; full.height = img.height
     const g = full.getContext('2d'); g.drawImage(img, 0, 0)
-    g.fillStyle = '#1f1f1d'; for (const r of masks) g.fillRect(Math.floor(r.x * scale), Math.floor(r.y * scale), Math.ceil(r.w * scale), Math.ceil(r.h * scale))
+    // masked personal data: a dark box with diagonal stripes, so a reader sees "hidden" and not a broken page (#64)
+    for (const r of masks) {
+      const x = Math.floor(r.x * scale); const y = Math.floor(r.y * scale); const w = Math.ceil(r.w * scale); const h = Math.ceil(r.h * scale)
+      g.fillStyle = '#1f1f1d'; g.fillRect(x, y, w, h)
+      g.save(); g.beginPath(); g.rect(x, y, w, h); g.clip(); g.strokeStyle = '#3d3d38'; g.lineWidth = 3 * scale
+      for (let i = -h; i < w; i += 12 * scale) { g.beginPath(); g.moveTo(x + i, y + h); g.lineTo(x + i + h, y); g.stroke() }
+      g.restore()
+    }
     for (const m of marks) {
       const r = 13 * scale; const x = Math.max(r + 2, m.x * scale); const y = Math.max(r + 2, m.y * scale)
       g.beginPath(); g.arc(x, y, r, 0, Math.PI * 2); g.fillStyle = '#ffe452'; g.fill(); g.lineWidth = 2 * scale; g.strokeStyle = '#151514'; g.stroke()
@@ -50,12 +58,15 @@ export async function stepSpec (page, tokens = null, mode = null) {
 
 export function captureOptions (scenario, node) {
   const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
+  // masks add up: a step's own selectors never drop the scenario's (personal data stays hidden on every step)
+  const masks = [maskList(scenario?.capture?.mask), maskList(node?.capture?.mask)]
   const q = Number(o.quality)
   return {
     format: o.format === 'png' ? 'png' : 'webp',
     scale: Number(o.scale) === 2 ? 2 : 1,
     quality: Number.isFinite(q) && q > 0 ? Math.min(1, Math.max(0.3, q)) : 0.82,
-    mask: Array.isArray(o.mask) ? o.mask.map(String).filter(Boolean) : [],
+    mask: [...new Set(masks.flatMap((m) => m.selectors))],
+    maskAuto: masks.some((m) => m.auto),
     annotate: o.annotate === true,
     // stack pages default to the first screen: hundreds of full-page shots would swamp storage
     fullPage: o.full_page != null ? o.full_page !== false : node?.kind !== 'stack'
@@ -121,6 +132,9 @@ export async function captureStep ({ page, node, locale, viewports, run, console
   const original = page.viewportSize()
   const results = []
   let netResults = null
+  // personal data (#64): what this capture masks; recipients of a captured e-mail are masked before the body
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  if (mask.active && extra.mail) extra = { ...extra, mail: maskMailHeaders(extra.mail, mask) }
   // batch E: the engine of this page, and per key its colour scheme, motion and display mode (run scope variants)
   const engine = engineOf(page)
   const scope = run.meta?.scope
@@ -128,14 +142,38 @@ export async function captureStep ({ page, node, locale, viewports, run, console
     const now = page.viewportSize()
     if (!now || now.width !== width || now.height !== height) await page.setViewportSize({ width, height })
     const undoVariant = await applyVariant(page, scope?.variants?.[viewport], scope?.devices?.[scope?.variants?.[viewport]?.device] || null)
+    let masking = null
     try {
       await settle(page)
       const html = paths.step(stepId, locale, viewport, 'html')
       await rm(paths.step(stepId, locale, viewport, 'eval.json'), { force: true })
       await mkdir(dirname(html), { recursive: true })
-      // PII selectors are covered by a solid box; at 2× Chrome renders the full page at double scale (CDP), and
-      // the boxes are drawn while encoding because Playwright's own mask only works at the page's pixel ratio
-      const maskRects = opts.mask.length ? await page.evaluate((sels) => sels.flatMap((sel) => { try { return [...document.querySelectorAll(sel)] } catch { return [] } }).map((el) => { const r = el.getBoundingClientRect(); return { x: r.left + scrollX, y: r.top + scrollY, w: r.width, h: r.height } }).filter((r) => r.w > 0 && r.h > 0), opts.mask) : []
+      // the checks read the page as it is: masks change what is stored, never a verdict
+      let result
+      for (let attempt = 0; ; attempt++) {
+        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
+          // the page navigated under us (redirect, Turbo); wait once and probe the new document
+          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
+          throw error
+        }
+      }
+      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
+      const ignore = await resolveIgnore(page, node, locale, viewport)
+      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the mask and the keyboard walk
+      await liveValueChecks({ page, node, result, url: page.url(), tokens })
+      // One mask for every output (#64): from here to the end of the capture the page shows placeholders or fake
+      // values instead of personal data, so the screenshot, HTML, outline, spec, keyboard shots and HTML view agree
+      if (mask.active) {
+        masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() }))
+        rememberPii(masking.pairs)
+        await page.evaluate((known) => window.__kalokoMask?.update({ known }), knownPii()).catch(() => {})
+        // the outline the evaluators read comes from the masked page; checks and boxes keep the first probe
+        if (masking.rects.length || masking.pairs.length || Object.keys(masking.counts).length) {
+          const again = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)).catch(() => null)
+          if (again) for (const k of ['state', 'scoped', 'overflow', 'meta', 'design']) if (again[k] !== undefined) result[k] = again[k]
+        }
+      }
+      const maskRects = (masking?.rects || []).filter((r) => r.box)
       let shot
       // a device preset's pixel ratio (3 on an iPhone) shapes the layout, not the image: captures stay in CSS pixels
       // (or 2× for documentation), so the canvas, boxes and diffs work the same for every device
@@ -152,22 +190,12 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         // WebKit and Firefox have no 2× capture of their own: 2× when the context renders at 2× or more, else 1×
         // (exactly 2×: a 3× or 2.625× device would give an image the masks and marks are not drawn for)
         if (opts.scale === 2 && dpr !== 2) shotScale = 1
-        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css', ...(opts.mask.length && shotScale === 1 ? { mask: opts.mask.map((sel) => page.locator(sel)), maskColor: '#1f1f1d' } : {}) })
+        shot = await page.screenshot({ fullPage: opts.fullPage, scale: shotScale === 2 ? 'device' : 'css' })
       }
       // secrets the CLI used (gate, accounts) never land in a capture, even if the page echoes them
       await writeFile(html, redact(await page.content()))
-      let result
-      for (let attempt = 0; ; attempt++) {
-        try { result = await page.evaluate(probe, probeArgs(node, maxChars, designSummary)); break } catch (error) {
-          // the page navigated under us (redirect, Turbo); wait once and probe the new document
-          if (attempt < 2 && /Execution context was destroyed|navigation/i.test(error.message)) { await page.waitForLoadState('domcontentloaded').catch(() => {}); await page.waitForTimeout(500); continue }
-          throw error
-        }
-      }
       // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
       if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens, scope?.variants?.[viewport]?.token_mode || scope?.variants?.[viewport]?.color_scheme || null)
-      // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
-      const ignore = await resolveIgnore(page, node, locale, viewport)
       // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
       const unstable = stability > 1 ? await repeatRegions(page, { n: Math.min(6, stability), fullPage: opts.fullPage }).catch((error) => ({ error: String(error.message).slice(0, 200) })) : null
       // numbered marks on the highlighted elements, in criteria order (first box of each)
@@ -176,7 +204,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       let thumb = paths.step(stepId, locale, viewport, 'thumb.webp')
       let thumbWritten = true
       try {
-        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: shotScale === 2 ? maskRects : [] })
+        // mask boxes are drawn while encoding, at the image's own scale (Playwright's mask only works at the page's pixel ratio)
+        const enc = await encodeImage(page, shot, { format: opts.format, quality: opts.quality, marks: annotations, scale: shotScale, masks: maskRects })
         // the extension follows the bytes (WebKit encodes PNG), so the service accepts the file
         if (enc.imageType === 'image/png') image = paths.step(stepId, locale, viewport, 'png')
         if (enc.thumbType === 'image/png') thumb = paths.step(stepId, locale, viewport, 'thumb.png')
@@ -184,13 +213,11 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         await writeFile(thumb, enc.thumb)
       } catch (error) {
         // never store an unmasked image when masks were asked for
-        if (shotScale === 2 && maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
+        if (maskRects.length) throw new Error(`capture: could not draw the masks on ${node.id} (${error.message}); the image was not stored`)
         image = paths.step(stepId, locale, viewport, 'png'); thumbWritten = false
         await writeFile(image, shot)
       }
       const url = page.url()
-      // URL, computed styles and read-only expressions (src/lib/assertions.mjs), before the keyboard walk moves focus
-      await liveValueChecks({ page, node, result, url, tokens })
       const collected = consoleFile ? await readJsonl(consoleFile) : consoleEntriesFor(page)
       const consoleEntries = (collected || []).filter((e) => !since || e.ts >= since)
       // requests blocked by the readonly policy surface as ERR_BLOCKED_BY_CLIENT; they are reported separately
@@ -236,7 +263,7 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       let view = null
       if (htmlView) {
         try {
-          // the same PII masks as the screenshot (#27), applied to the text of the copy
+          // the page is still masked here (#64); the selectors are covered in the copy as well (#27)
           const rev = await buildHtmlView({ page, run, scenarioId: htmlView.scenarioId, node: node.id, locale, viewport, mask: [...(opts.mask || []), ...(htmlView.mask || [])] })
           await writeJson(join(run.dir, 'views', `${rev.id}.json`), rev)
           view = { revision: rev.id, entry: rev.entry }
@@ -254,6 +281,8 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         ...extra,
         http_status: extra.http_status ?? result.http_status ?? null,
         capture: { format: String(image).split('.').pop(), scale: shotScale, ...(dpr !== 1 ? { dpr } : {}), masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+        // what was hidden and where (#64): the canvas marks these places, so a blank is not read as a bug
+        ...(mask.active ? { pii: { mode: mask.mode, auto: mask.auto, selectors: mask.selectors.length, kinds: masking?.counts || {}, rects: (masking?.rects || []).slice(0, 200).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) } } : {}),
         ...(ignore.length ? { ignore } : {}),
         ...(unstable ? { unstable } : {}),
         ...(keyboard ? { keyboard } : {}),
@@ -265,27 +294,42 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       await writeJson(file, redactDeep(record))
       await addCapture(run.dir, { type: 'capture', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, file: relative(run.dir, file), at: record.captured_at })
       results.push(record)
-    } finally { await undoVariant() }
+    } finally {
+      // the page gets its own text back, also when the capture failed half-way
+      if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {})
+      await undoVariant()
+    }
   }
   const last = page.viewportSize()
   if (original && (!last || last.width !== original.width || last.height !== original.height)) await page.setViewportSize(original)
   return results
 }
 
-export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0 }) {
+export async function captureCrop ({ page, node, locale, selector, title, run, padding = 0, options = null }) {
   const paths = runPaths(run.dir)
   const el = page.locator(selector).first()
   const name = `${node.id}-${locale}-${Date.now().toString(36)}.png`
   const file = paths.attachment(name)
   await mkdir(dirname(file), { recursive: true })
-  if (padding > 0) {
-    // element plus a margin of surrounding page, clipped from a full-page shot (documentation crops)
+  // the same mask as the step's capture (#64): masked text in the page, boxes over masked elements
+  const opts = options || captureOptions(null, node)
+  const mask = maskSpec({ run, selectors: opts.mask, auto: opts.maskAuto, locale })
+  let masking = null
+  try {
+    if (mask.active) { masking = await page.evaluate(inPage(maskInPage, { ...mask, known: knownPii() })); rememberPii(masking.pairs) }
+    const boxes = (masking?.rects || []).filter((r) => r.box)
+    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
     const box = await el.boundingBox()
     if (!box) throw new Error(`crop: ${selector} is not visible`)
-    const doc = await page.evaluate(() => ({ w: document.documentElement.scrollWidth, h: document.documentElement.scrollHeight, x: scrollX, y: scrollY }))
+    // element plus a margin of surrounding page (documentation crops), clipped from a full-page shot
     const x = Math.max(0, box.x + doc.x - padding); const y = Math.max(0, box.y + doc.y - padding)
-    await page.screenshot({ path: file, fullPage: true, clip: { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) } })
-  } else await el.screenshot({ path: file })
+    const clip = { x, y, width: Math.min(doc.w - x, box.width + padding * 2), height: Math.min(doc.h - y, box.height + padding * 2) }
+    const png = padding > 0 || boxes.length ? await page.screenshot({ fullPage: true, clip }) : await el.screenshot()
+    if (boxes.length) {
+      const enc = await encodeImage(page, png, { format: 'png', masks: boxes.map((r) => ({ ...r, x: r.x - clip.x, y: r.y - clip.y })) })
+      await writeFile(file, enc.image)
+    } else await writeFile(file, png)
+  } finally { if (masking) await page.evaluate(() => window.__kalokoMaskUndo?.()).catch(() => {}) }
   const entry = { type: 'attachment', kind: 'crop', node: node.id, locale, title: title || selector, selector, file: relative(run.dir, file), viewport: page.viewportSize(), at: new Date().toISOString() }
   await addCapture(run.dir, entry)
   return entry

package/src/lib/doctor.mjs

@@ -237,7 +237,8 @@ async function defaultInstall (engine) {
 
 async function defaultEngine (engine) {
   const pw = await import('playwright')
-  const browser = await pw[engine].launch({ headless: true })
+  const { launchWithRetry } = await import('./browser.mjs')
+  const browser = await launchWithRetry(pw[engine], { headless: true })
   const version = browser.version()
   await browser.close()
   return { version }
@@ -245,7 +246,8 @@ async function defaultEngine (engine) {
 
 async function defaultLaunch (config) {
   const { chromium } = await import('playwright')
-  const browser = await chromium.launch({ headless: true, channel: config.browser?.channel || undefined })
+  const { launchWithRetry } = await import('./browser.mjs')
+  const browser = await launchWithRetry(chromium, { headless: true, channel: config.browser?.channel || undefined })
   const version = browser.version()
   await browser.close()
   return { version }

package/src/lib/evaluate.mjs

@@ -4,6 +4,8 @@ import { readCaptures, runPaths } from './run.mjs'
 import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
 import { visionSettings, visionCriteria, templateReason, VISION_PROMPT_VERSION } from './vision-core.mjs'
 import { prepareImages, visionPlan, askVision, hostedVisionSender } from './vision.mjs'
+import { triageSettings, probeFindings, TRIAGE_PROMPT_VERSION } from './triage-core.mjs'
+import { askTriage, hostedTriageSender, chooseFrames, frameBatches, mergeFindings } from './triage.mjs'
 import { KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
 import { packHelp } from './pack-help.mjs'
@@ -31,7 +33,7 @@ export const evaluationSource = (node, capture) => sha256({ capture, criteria: n
  * asked, the vision evaluator. Every semantic verdict with a score carries a one-sentence `reason`: the vision
  * model's own, or a sentence saying what the text evaluator read (it returns a probability only).
  */
-export function combine ({ node, capture, jev, vision = null, thresholds, environment = null }) {
+export function combine ({ node, capture, jev, vision = null, triage = null, cascade = false, thresholds, environment = null }) {
   return (node.criteria || []).map((c) => {
     if (!appliesOn(c, environment)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies on ${c.environments.join(', ')} only; this walk ran on ${environment}.`, na: true, not_applicable: 'environment' }
     if (!appliesToState(c, capture?.step_state)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies to the state${c.states.length > 1 ? 's' : ''} ${c.states.join(', ')} only; this capture shows ${capture?.step_state || 'default'}.`, na: true, not_applicable: 'state' }
@@ -48,15 +50,19 @@ export function combine ({ node, capture, jev, vision = null, thresholds, enviro
       const answer = jev?.response?.answers?.[c.id]
       const text = answer?.noul ?? null
       const seen = vision?.answers?.[c.id] || null
-      const score = seen ? seen.probability : text
+      // Clef's answer on the screenshot (#71): recorded beside the others; with cascade on it decides when it is sure
+      const clef = triage?.criteria?.[c.id] ?? null
+      const clefDecides = cascade && !seen && clef != null && (clef <= thresholds.fail || clef >= thresholds.pass)
+      const score = seen ? seen.probability : clefDecides ? clef : text
       const verdict = verdictFor(score, thresholds)
-      const reason = seen?.reason || templateReason({ score, scope: c.check.scope, about: c.check.about, verdict })
+      const reason = seen?.reason || (clefDecides ? `Judged from the screenshot by Clef: ${Math.round(clef * 100)} % yes, clear enough for a verdict.` : templateReason({ score, scope: c.check.scope, about: c.check.about, verdict }))
       return {
         id: c.id, type: 'semantic', verdict, evidence: null, score,
-        provider: seen ? `${vision.provider || 'anthropic'}-vision` : answer ? 'typesafe-jev' : null,
+        provider: seen ? `${vision.provider || 'anthropic'}-vision` : clefDecides ? 'cloudflare-clef' : answer ? 'typesafe-jev' : null,
         note: score == null ? (jev?.skipped || jev?.error || vision?.skipped || vision?.error || null) : null,
         ...(reason ? { reason, reason_source: seen?.reason ? 'model' : 'template' } : {}),
-        ...(seen && text != null ? { text_score: text } : {}),
+        ...((seen || clefDecides) && text != null ? { text_score: text } : {}),
+        ...(clef != null ? { clef_score: clef } : {}),
         ...(!seen && (vision?.skipped || vision?.error) && vision.ids?.includes(c.id) ? { vision_note: vision.skipped || vision.error } : {})
       }
     }
@@ -80,12 +86,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   // with an own text-evaluator key the vision evaluator uses an own key too, unless hosted: true says otherwise
   if (vset && process.env[settings.api_key_env] && config.evaluators?.vision?.hosted !== true) vset.hosted = false
   const vision = vset ? visionContext({ config, root, settings: vset, report, fetchImpl }) : null
+  // triage (#65, experimental): the kind of problem on each screen and in recording frames, advisory only
+  const tset = triageSettings(config)
+  if (tset) report.triage = { model: tset.model, screens: 0, frames: 0, calls: 0, cached: 0, findings: 0, shown: 0, skipped: null, via: {}, ms: [] }
+  const triage = tset ? triageContext({ config, root, settings: tset, report, fetchImpl, uncertain: (vset || {}).uncertain }) : null
   // checks against the design reference and the token set (pack design, #42) need the service and a browser
   const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
   // style checks against a token (style_equals token: color.brand.primary) use the project's token set
   const styleTokens = scenario.nodes.some((n) => tokenCriteria(n).length) ? (drift ? { tokens: drift.tokens, mode: drift.mode } : await runTokens({ config, root })) : null
   try {
-  const latest = latestCaptures(await readCaptures(run.dir))
+  const entries = await readCaptures(run.dir)
+  const latest = latestCaptures(entries)
   // stack steps: semantic criteria only on the first N pages (items.semantic_sample, default 20); cost stays bounded
   const sampled = new Map()
   for (const e of latest) {
@@ -129,13 +140,15 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
       }
     }))
-    // second opinion with the screenshot for flagged and uncertain criteria
+    // what kind of problem each screen and its recording shows; with cascade on, Clef may settle uncertain criteria
+    if (triage) await triage.ask(work.filter((w) => w.current && !w.outOfSample), run.dir, entries)
+    // second opinion with the screenshot for flagged and uncertain criteria (those Clef did not settle)
     if (vision) await vision.ask(work.filter((w) => w.current && !w.outOfSample && w.request), run.dir)
-    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen } of work) {
+    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen, triage: tri } of work) {
       const source = evaluationSource(node, capture)
       if (drift && current && needsDrift(node)) { capture.checks = { ...capture.checks, ...await driftForCapture({ node, entry, capture, runDir: run.dir, ctx: drift }) }; report.drift++ }
       if (styleTokens?.tokens && current && tokenCriteria(node).length) capture.checks = { ...capture.checks, ...storedValueChecks(node, capture, { ...styleTokens, recorded: !!capture.rechecked_at, only: (c) => !!c.check.token && STYLE_ASSERTS.has(c.check.assert) }) }
-      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, thresholds: settings.thresholds, environment })
+      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, triage: tri, cascade: !!tset?.cascade, thresholds: settings.thresholds, environment })
       // pages outside the semantic sample carry only the deterministic results
       if (outOfSample) criteria = criteria.filter((c) => c.type !== 'semantic' || c.evidence === 'no console errors')
       const evaluation = {
@@ -156,6 +169,8 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         thresholds: settings.thresholds,
         state_chars: request ? JSON.stringify(request.state).length : null,
         ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
+        // advisory findings (#65, #66): the kind of problem, never a verdict
+        ...(tri ? { triage: tri.findings ? { provider: 'cloudflare-clef', via: tri.via, model: tri.model, prompt_version: TRIAGE_PROMPT_VERSION, requests: tri.requests, cached: tri.cached, ms: tri.ms } : { provider: 'cloudflare-clef', skipped: tri.skipped || null, error: tri.error || null }, ...(tri.findings?.length ? { findings: tri.findings } : {}) } : {}),
         ...(seen ? { vision: seen.answers ? { provider: seen.provider, via: seen.via, model: seen.model, prompt_version: VISION_PROMPT_VERSION, cached: seen.cached, usage: seen.usage || null, answers: seen.answers } : { provider: seen.provider, skipped: seen.skipped || null, error: seen.error || null } } : {}),
         criteria
       }
@@ -165,8 +180,11 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
       for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes${c.reason_source === 'model' ? `: ${c.reason}` : ''}` : c.note) || null, ...(c.reason ? { reason: c.reason } : {}), ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
     }
   }
-  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close() }
-  const { hostedStop, visionStop, ...out } = report
+  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close(); await triage?.close() }
+  // triage is on by default but stays silent where nobody asked for it and it did not run (experiment off, no service)
+  if (report.triage && !report.triage.screens && config.evaluators?.triage == null) delete report.triage
+  if (report.triage) { const ms = report.triage.ms.filter((x) => x != null).sort((a, b) => a - b); report.triage.median_ms = ms.length ? ms[Math.floor(ms.length / 2)] : null; delete report.triage.ms }
+  const { hostedStop, visionStop, triageStop, ...out } = report
   return { ...out, outdated: [...report.outdated] }
 }
 
@@ -249,7 +267,7 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
       const queue = []
       for (const w of work) {
         const semantic = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && w.request.questions[c.id])
-        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings })
+        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings }).filter((c) => !w.triageDecided?.has(c.id))
         if (crit.length) queue.push({ w, crit })
       }
       await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 3, queue.length)) }, async () => {
@@ -281,6 +299,88 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
   }
 }
 
+/**
+ * The triage step of an evaluation (#65): one Clef request per screen (page tiles), and for a step with a recording
+ * up to `frames_per_step` chosen frames in requests of four. Uncertain semantic criteria ride along as extra
+ * questions (#71). Fills `w.triage` with merged findings; a stopped service (experiment off, allowance) ends triage
+ * for this evaluation with one note in the report.
+ */
+function triageContext ({ config, root, settings, report, fetchImpl, uncertain = [0.15, 0.85] }) {
+  const cacheDir = join(resolve(root, config.output), '.cache', 'triage')
+  const hosted = hostedTriageSender(config, report)
+  let browser = null; let pagePromise = null
+  const page = () => (pagePromise ||= (async () => {
+    const { launchEphemeral } = await import('./browser.mjs')
+    browser = await launchEphemeral(config)
+    const p = await browser.newPage()
+    await p.setContent('<!doctype html><title>triage</title>')
+    return p
+  })())
+  const allowCall = () => { if (report.triage.calls >= (settings.max_calls ?? 200)) return false; report.triage.calls++; return true }
+  const own = process.env.CLOUDFLARE_API_TOKEN && process.env.CLOUDFLARE_ACCOUNT_ID
+  const unavailable = !own && !process.env[config.share?.token_env || 'KALOKO_TOKEN'] ? 'no service token (KALOKO_TOKEN)' : null
+  let framesLeft = settings.frames_per_run
+  const ask = async (w, payload, where) => {
+    const r = await askTriage({ payload, settings, cacheDir, where, fetchImpl, hosted, allowCall })
+    if (r.findings) { if (r.cached) report.triage.cached++; report.triage.via[r.via] = (report.triage.via[r.via] || 0) + 1; report.triage.ms.push(r.ms) }
+    else if (!report.triage.skipped) report.triage.skipped = r.skipped || r.error
+    return r
+  }
+  return {
+    async ask (work, runDir, entries) {
+      if (unavailable) { report.triage.skipped = unavailable; return }
+      if (!own && !(await hosted.ready())) { report.triage.skipped = report.triageStop; return }
+      const queue = [...work]
+      await Promise.all(Array.from({ length: Math.max(1, Math.min(3, queue.length)) }, async () => {
+        for (let w = queue.shift(); w; w = queue.shift()) {
+          if (report.triageStop) break
+          const file = w.capture.files?.image || w.capture.files?.png
+          if (!file) continue
+          try {
+            const [lo, hi] = uncertain
+            const criteria = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && c.check.about !== 'console' && w.request?.questions?.[c.id])
+              .filter((c) => { const s = w.jev?.response?.answers?.[c.id]?.noul; return s != null && s > lo && s < hi })
+              .map((c) => ({ id: c.id, question: c.check.question }))
+            const context = { purpose: purposeOf(w.node, w.capture), url: w.capture.url, locale: w.capture.locale, viewport: w.entry.viewport, signals: { overflow: (w.capture.overflow || []).length, console_errors: (w.capture.console_errors || []).length } }
+            const { images } = await prepareImages({ page: await page(), file: join(runDir, file), cssWidth: w.capture.size?.width, full: true, settings: { max_width: settings.max_width, tile_height: settings.tile_height, max_tiles: settings.max_tiles } })
+            const tiles = images.slice(0, 4).map((i) => `data:${i.media_type};base64,${i.data}`)
+            report.triage.screens++
+            const lists = []; const results = []
+            const screen = await ask(w, { kind: 'screen', context, criteria, images: tiles }, {})
+            results.push(screen)
+            if (screen.findings) lists.push(screen.findings)
+            // the recording of this capture (same walk), as frames
+            const rec = entries.filter((e) => e.type === 'recording' && e.node === w.entry.node && e.locale === w.entry.locale && e.viewport === w.entry.viewport && e.captured_at && e.captured_at === w.capture.captured_at).pop()
+            if (rec && framesLeft > 0 && !report.triageStop) {
+              const side = await readJson(join(runDir, rec.file), null)
+              const video = side?.files?.webm || side?.files?.mp4
+              if (video) {
+                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.max_width })
+                framesLeft -= frames.length; report.triage.frames += frames.length
+                for (const batch of frameBatches(frames)) {
+                  const r = await ask(w, { kind: 'motion', context: { ...context, frames: batch.map((f) => ({ t_ms: f.t, why: f.why })) }, images: batch.map((f) => f.data) }, { frames: batch.map((f) => f.t) })
+                  results.push(r)
+                  if (r.findings) lists.push(r.findings)
+                }
+              }
+            }
+            const done = results.filter((r) => r.findings)
+            if (!done.length) { w.triage = { skipped: results[0]?.skipped || null, error: results[0]?.error || null }; continue }
+            const findings = mergeFindings([...lists, probeFindings(w.capture)])
+            report.triage.findings += findings.length; report.triage.shown += findings.filter((f) => f.shown).length
+            w.triage = { findings, criteria: screen.criteria || {}, model: done[0].model, via: done[0].via, cached: done.every((r) => r.cached), requests: done.length, ms: done.reduce((n, r) => n + (r.ms || 0), 0) || null }
+            // cascade (#71): criteria Clef is sure about skip the vision evaluator
+            if (settings.cascade) w.triageDecided = new Set(Object.entries(screen.criteria || {}).filter(([, p]) => p <= 0.15 || p >= 0.85).map(([id]) => id))
+          } catch (error) {
+            w.triage = { error: String(error.message || error).slice(0, 300) }
+          }
+        }
+      }))
+    },
+    async close () { if (browser) await browser.close().catch(() => {}) }
+  }
+}
+
 /**
  * Sends a JEV request through the organization's service (hosted evaluation) when a service token is set.
  * An allowance used up or a service without hosting stops further hosted calls for this run with one message.

package/src/lib/mail.mjs

@@ -1,5 +1,5 @@
 // E-mails of a flow, from wherever the project's stack puts them. Every source returns the same
-// message shape: { id, subject, from, to: [address], date, html, text }.
+// message shape: { id, subject, from, to: [address], cc: [address], date, html, text }.
 //   mailpit — built in (REST API), optional gate auth;
 //   script  — a project module for anything else (letter_opener, Mailosaur, IMAP, the app's own API);
 //   file    — `kaloko mail --file`: a message the agent obtained itself (.eml, .html, .json, .txt).
@@ -40,7 +40,7 @@ export function mailpit (mail) {
         hitId = hit.ID
       }
       const m = await get(`/api/v1/message/${encodeURIComponent(hitId)}`)
-      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
+      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), cc: (m.Cc || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
     }
   }
 }
@@ -95,6 +95,7 @@ export function normalize (m) {
     subject: m.subject ?? m.Subject ?? '',
     from: typeof from === 'object' && from ? address(from) : from || '',
     to: list(m.to ?? (m.To || []).map?.((t) => t.Address ?? t)),
+    cc: list(m.cc ?? (m.Cc || []).map?.((t) => t.Address ?? t)),
     date: m.date ?? m.Date ?? null,
     html: m.html ?? m.HTML ?? null,
     text: m.text ?? m.Text ?? null
@@ -109,8 +110,8 @@ export async function readMessageFile (file) {
   if (ext === '.eml' || ext === '.msg' || ext === '.mbox') return parseEml(raw)
   if (ext === '.json') return normalize(JSON.parse(raw.toString('utf8')))
   const body = raw.toString('utf8')
-  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], date: null, html: body, text: null }
-  return { id: '', subject: '', from: '', to: [], date: null, html: null, text: body }
+  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], cc: [], date: null, html: body, text: null }
+  return { id: '', subject: '', from: '', to: [], cc: [], date: null, html: null, text: body }
 }
 
 // --- RFC 5322 / MIME, enough for what mailers send: multipart, base64, quoted-printable, charsets ---
@@ -175,6 +176,7 @@ export function parseEml (raw) {
     subject: h('subject'),
     from: h('from'),
     to: list(h('to')),
+    cc: list(h('cc')),
     date: top.headers.date ? new Date(top.headers.date).toISOString() : null,
     html: found.html ?? null,
     text: found.text ?? null

package/src/lib/manifest-core.mjs

@@ -63,10 +63,21 @@ export function slimCapture (cap) {
     ...(cap.design ? { design: cap.design } : {}), ...(cap.capture ? { capture: cap.capture } : {}),
     ...(cap.criteria_hash ? { criteria_hash: cap.criteria_hash } : {}),
     ...Object.fromEntries(DIM_FIELDS.filter((k) => cap[k]).map((k) => [k, cap[k]])),
-    ...(cap.ignore?.length ? { ignore: cap.ignore } : {}), ...(cap.unstable ? { unstable: cap.unstable } : {}), ...(cap.keyboard ? { keyboard: cap.keyboard } : {})
+    ...(cap.ignore?.length ? { ignore: cap.ignore } : {}), ...(cap.unstable ? { unstable: cap.unstable } : {}), ...(cap.keyboard ? { keyboard: cap.keyboard } : {}),
+    ...(cap.pii ? { pii: cap.pii } : {})
   }
 }
 
+/** Masked personal data of a capture as the canvas marks it (#64): mode, kinds found and up to 200 places. */
+export const piiOf = (p) => ({ mode: p.mode || 'redact', auto: !!p.auto, kinds: p.kinds || {}, rects: (p.rects || []).slice(0, 200) })
+
+/** The run's flag "personal data hidden" (#64): set when the run masks automatically or any capture was masked. */
+export function runPii (meta, nodes) {
+  const masked = nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((byVp) => Object.values(byVp || {}))).filter((c) => c?.pii)
+  if (!meta?.pii && !masked.length) return null
+  return { hidden: true, mode: meta?.pii?.mode || masked[0]?.pii?.mode || 'redact', auto: !!meta?.pii, captures: masked.length }
+}
+
 /** The keyboard walk of a capture as the canvas draws it: tab stops (numbered boxes) and the focus screenshots. */
 export const keyboardOf = (k) => k ? { complete: !!k.complete, total: k.total ?? (k.stops || []).length, stops: (k.stops || []).slice(0, 40), shots: (k.shots || []).slice(0, 10) } : null
 
@@ -100,12 +111,16 @@ export function assembleNode ({ node, pos, locales, viewportNames: allKeys, capt
       mail: cap.mail || null, eval: ev ? c.files.eval : null, ...(c.files.spec ? { spec: c.files.spec } : {}),
       // quick signals from the evaluator (probabilities: purpose met, expected language, looks broken)
       ...(ev?.signals ? { signals: ev.signals } : {}),
+      // what kind of problem Clef saw (#65, experimental): advisory findings, never a verdict
+      ...(ev?.findings?.length ? { findings: ev.findings.slice(0, 30) } : {}),
       ...(c.live ? { live: c.live } : {}),
       ...(cap.view?.revision ? { view: cap.view } : {}),
       // regions left out of pixel diffs (#20), regions that moved between repeated shots, the keyboard walk (#19)
       ...(cap.ignore?.length ? { ignore: cap.ignore.slice(0, 50) } : {}),
       ...(cap.unstable?.regions?.length ? { unstable: cap.unstable.regions.slice(0, 50) } : {}),
-      ...(cap.keyboard ? { keyboard: keyboardOf(cap.keyboard) } : {})
+      ...(cap.keyboard ? { keyboard: keyboardOf(cap.keyboard) } : {}),
+      // personal data hidden on this capture (#64): mode and the places the canvas marks
+      ...(cap.pii ? { pii: piiOf(cap.pii) } : {})
     }
     if (ev?.semantic || ev?.jev) semanticByKey[`${c.locale}|${c.viewport}`] = ev.semantic || ev.jev
     if (cap.metadata && !metadata[c.locale]) metadata[c.locale] = cap.metadata