kaloko 0.11.0 → 0.12.0

package/src/lib/evaluate.mjs

@@ -4,6 +4,8 @@ import { readCaptures, runPaths } from './run.mjs'
 import { buildRequest, evaluateWithJev, verdictFor, DEFAULTS, PROMPT_VERSION, SIGNAL_NAMES, purposeOf } from './jev.mjs'
 import { visionSettings, visionCriteria, templateReason, VISION_PROMPT_VERSION } from './vision-core.mjs'
 import { prepareImages, visionPlan, askVision, hostedVisionSender } from './vision.mjs'
+import { triageSettings, probeFindings, TRIAGE_PROMPT_VERSION } from './triage-core.mjs'
+import { askTriage, hostedTriageSender, chooseFrames, frameBatches, mergeFindings } from './triage.mjs'
 import { KEYBOARD_ASSERTS } from './keyboard.mjs'
 import { needsDrift, driftForCapture, loadTokens, driftCacheDir } from './drift.mjs'
 import { packHelp } from './pack-help.mjs'
@@ -31,7 +33,7 @@ export const evaluationSource = (node, capture) => sha256({ capture, criteria: n
  * asked, the vision evaluator. Every semantic verdict with a score carries a one-sentence `reason`: the vision
  * model's own, or a sentence saying what the text evaluator read (it returns a probability only).
  */
-export function combine ({ node, capture, jev, vision = null, thresholds, environment = null }) {
+export function combine ({ node, capture, jev, vision = null, triage = null, cascade = false, thresholds, environment = null }) {
   return (node.criteria || []).map((c) => {
     if (!appliesOn(c, environment)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies on ${c.environments.join(', ')} only; this walk ran on ${environment}.`, na: true, not_applicable: 'environment' }
     if (!appliesToState(c, capture?.step_state)) return { id: c.id, type: c.check.type, verdict: 'not_evaluated', evidence: `Applies to the state${c.states.length > 1 ? 's' : ''} ${c.states.join(', ')} only; this capture shows ${capture?.step_state || 'default'}.`, na: true, not_applicable: 'state' }
@@ -48,15 +50,19 @@ export function combine ({ node, capture, jev, vision = null, thresholds, enviro
       const answer = jev?.response?.answers?.[c.id]
       const text = answer?.noul ?? null
       const seen = vision?.answers?.[c.id] || null
-      const score = seen ? seen.probability : text
+      // Clef's answer on the screenshot (#71): recorded beside the others; with cascade on it decides when it is sure
+      const clef = triage?.criteria?.[c.id] ?? null
+      const clefDecides = cascade && !seen && clef != null && (clef <= thresholds.fail || clef >= thresholds.pass)
+      const score = seen ? seen.probability : clefDecides ? clef : text
       const verdict = verdictFor(score, thresholds)
-      const reason = seen?.reason || templateReason({ score, scope: c.check.scope, about: c.check.about, verdict })
+      const reason = seen?.reason || (clefDecides ? `Judged from the screenshot by Clef: ${Math.round(clef * 100)} % yes, clear enough for a verdict.` : templateReason({ score, scope: c.check.scope, about: c.check.about, verdict }))
       return {
         id: c.id, type: 'semantic', verdict, evidence: null, score,
-        provider: seen ? `${vision.provider || 'anthropic'}-vision` : answer ? 'typesafe-jev' : null,
+        provider: seen ? `${vision.provider || 'anthropic'}-vision` : clefDecides ? 'cloudflare-clef' : answer ? 'typesafe-jev' : null,
         note: score == null ? (jev?.skipped || jev?.error || vision?.skipped || vision?.error || null) : null,
         ...(reason ? { reason, reason_source: seen?.reason ? 'model' : 'template' } : {}),
-        ...(seen && text != null ? { text_score: text } : {}),
+        ...((seen || clefDecides) && text != null ? { text_score: text } : {}),
+        ...(clef != null ? { clef_score: clef } : {}),
         ...(!seen && (vision?.skipped || vision?.error) && vision.ids?.includes(c.id) ? { vision_note: vision.skipped || vision.error } : {})
       }
     }
@@ -80,12 +86,17 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
   // with an own text-evaluator key the vision evaluator uses an own key too, unless hosted: true says otherwise
   if (vset && process.env[settings.api_key_env] && config.evaluators?.vision?.hosted !== true) vset.hosted = false
   const vision = vset ? visionContext({ config, root, settings: vset, report, fetchImpl }) : null
+  // triage (#65, experimental): the kind of problem on each screen and in recording frames, advisory only
+  const tset = triageSettings(config)
+  if (tset) report.triage = { model: tset.model, screens: 0, frames: 0, calls: 0, cached: 0, findings: 0, shown: 0, skipped: null, via: {}, ms: [] }
+  const triage = tset ? triageContext({ config, root, settings: tset, report, fetchImpl, uncertain: (vset || {}).uncertain }) : null
   // checks against the design reference and the token set (pack design, #42) need the service and a browser
   const drift = scenario.nodes.some(needsDrift) ? await driftContext({ run, scenario, config, root }) : null
   // style checks against a token (style_equals token: color.brand.primary) use the project's token set
   const styleTokens = scenario.nodes.some((n) => tokenCriteria(n).length) ? (drift ? { tokens: drift.tokens, mode: drift.mode } : await runTokens({ config, root })) : null
   try {
-  const latest = latestCaptures(await readCaptures(run.dir))
+  const entries = await readCaptures(run.dir)
+  const latest = latestCaptures(entries)
   // stack steps: semantic criteria only on the first N pages (items.semantic_sample, default 20); cost stays bounded
   const sampled = new Map()
   for (const e of latest) {
@@ -129,13 +140,15 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         if ((w.jev.skipped || w.jev.error) && w.asks) report.skipped.push(`${w.entry.node}/${w.entry.locale}/${w.entry.viewport}: ${w.jev.skipped || w.jev.error}`)
       }
     }))
-    // second opinion with the screenshot for flagged and uncertain criteria
+    // what kind of problem each screen and its recording shows; with cascade on, Clef may settle uncertain criteria
+    if (triage) await triage.ask(work.filter((w) => w.current && !w.outOfSample), run.dir, entries)
+    // second opinion with the screenshot for flagged and uncertain criteria (those Clef did not settle)
     if (vision) await vision.ask(work.filter((w) => w.current && !w.outOfSample && w.request), run.dir)
-    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen } of work) {
+    for (const { entry, node, capture, current, outOfSample, request, jev, vision: seen, triage: tri } of work) {
       const source = evaluationSource(node, capture)
       if (drift && current && needsDrift(node)) { capture.checks = { ...capture.checks, ...await driftForCapture({ node, entry, capture, runDir: run.dir, ctx: drift }) }; report.drift++ }
       if (styleTokens?.tokens && current && tokenCriteria(node).length) capture.checks = { ...capture.checks, ...storedValueChecks(node, capture, { ...styleTokens, recorded: !!capture.rechecked_at, only: (c) => !!c.check.token && STYLE_ASSERTS.has(c.check.assert) }) }
-      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, thresholds: settings.thresholds, environment })
+      let criteria = combine({ node, capture: current ? capture : {}, jev, vision: seen, triage: tri, cascade: !!tset?.cascade, thresholds: settings.thresholds, environment })
       // pages outside the semantic sample carry only the deterministic results
       if (outOfSample) criteria = criteria.filter((c) => c.type !== 'semantic' || c.evidence === 'no console errors')
       const evaluation = {
@@ -156,6 +169,8 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
         thresholds: settings.thresholds,
         state_chars: request ? JSON.stringify(request.state).length : null,
         ...(signalsOf(jev) ? { signals: signalsOf(jev) } : {}),
+        // advisory findings (#65, #66): the kind of problem, never a verdict
+        ...(tri ? { triage: tri.findings ? { provider: 'cloudflare-clef', via: tri.via, model: tri.model, prompt_version: TRIAGE_PROMPT_VERSION, requests: tri.requests, cached: tri.cached, ms: tri.ms } : { provider: 'cloudflare-clef', skipped: tri.skipped || null, error: tri.error || null }, ...(tri.findings?.length ? { findings: tri.findings } : {}) } : {}),
         ...(seen ? { vision: seen.answers ? { provider: seen.provider, via: seen.via, model: seen.model, prompt_version: VISION_PROMPT_VERSION, cached: seen.cached, usage: seen.usage || null, answers: seen.answers } : { provider: seen.provider, skipped: seen.skipped || null, error: seen.error || null } } : {}),
         criteria
       }
@@ -165,8 +180,11 @@ export async function evaluateRun ({ run, scenario, config, root, fetchImpl }) {
       for (const c of criteria) if (c.verdict === 'fail' || c.verdict === 'needs_review') report.findings.push({ node: entry.node, ...(entry.item ? { item: entry.item } : {}), locale: entry.locale, viewport: entry.viewport, id: c.id, verdict: c.verdict, type: c.type, why: c.evidence || (c.score != null ? `AI ${Math.round(c.score * 100)} % yes${c.reason_source === 'model' ? `: ${c.reason}` : ''}` : c.note) || null, ...(c.reason ? { reason: c.reason } : {}), ...((node.criteria || []).find((x) => x.id === c.id)?.pack && packHelp(c.id) ? { fix: packHelp(c.id).fix } : {}) })
     }
   }
-  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close() }
-  const { hostedStop, visionStop, ...out } = report
+  } finally { if (drift?.browser) await drift.browser.close(); await vision?.close(); await triage?.close() }
+  // triage is on by default but stays silent where nobody asked for it and it did not run (experiment off, no service)
+  if (report.triage && !report.triage.screens && config.evaluators?.triage == null) delete report.triage
+  if (report.triage) { const ms = report.triage.ms.filter((x) => x != null).sort((a, b) => a - b); report.triage.median_ms = ms.length ? ms[Math.floor(ms.length / 2)] : null; delete report.triage.ms }
+  const { hostedStop, visionStop, triageStop, ...out } = report
   return { ...out, outdated: [...report.outdated] }
 }
 
@@ -249,7 +267,7 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
       const queue = []
       for (const w of work) {
         const semantic = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && w.request.questions[c.id])
-        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings })
+        const crit = visionCriteria({ criteria: semantic, answers: w.jev?.response?.answers, settings }).filter((c) => !w.triageDecided?.has(c.id))
         if (crit.length) queue.push({ w, crit })
       }
       await Promise.all(Array.from({ length: Math.max(1, Math.min(settings.concurrency || 3, queue.length)) }, async () => {
@@ -281,6 +299,88 @@ function visionContext ({ config, root, settings, report, fetchImpl }) {
   }
 }
 
+/**
+ * The triage step of an evaluation (#65): one Clef request per screen (page tiles), and for a step with a recording
+ * up to `frames_per_step` chosen frames in requests of four. Uncertain semantic criteria ride along as extra
+ * questions (#71). Fills `w.triage` with merged findings; a stopped service (experiment off, allowance) ends triage
+ * for this evaluation with one note in the report.
+ */
+function triageContext ({ config, root, settings, report, fetchImpl, uncertain = [0.15, 0.85] }) {
+  const cacheDir = join(resolve(root, config.output), '.cache', 'triage')
+  const hosted = hostedTriageSender(config, report)
+  let browser = null; let pagePromise = null
+  const page = () => (pagePromise ||= (async () => {
+    const { launchEphemeral } = await import('./browser.mjs')
+    browser = await launchEphemeral(config)
+    const p = await browser.newPage()
+    await p.setContent('<!doctype html><title>triage</title>')
+    return p
+  })())
+  const allowCall = () => { if (report.triage.calls >= (settings.max_calls ?? 200)) return false; report.triage.calls++; return true }
+  const own = process.env.CLOUDFLARE_API_TOKEN && process.env.CLOUDFLARE_ACCOUNT_ID
+  const unavailable = !own && !process.env[config.share?.token_env || 'KALOKO_TOKEN'] ? 'no service token (KALOKO_TOKEN)' : null
+  let framesLeft = settings.frames_per_run
+  const ask = async (w, payload, where) => {
+    const r = await askTriage({ payload, settings, cacheDir, where, fetchImpl, hosted, allowCall })
+    if (r.findings) { if (r.cached) report.triage.cached++; report.triage.via[r.via] = (report.triage.via[r.via] || 0) + 1; report.triage.ms.push(r.ms) }
+    else if (!report.triage.skipped) report.triage.skipped = r.skipped || r.error
+    return r
+  }
+  return {
+    async ask (work, runDir, entries) {
+      if (unavailable) { report.triage.skipped = unavailable; return }
+      if (!own && !(await hosted.ready())) { report.triage.skipped = report.triageStop; return }
+      const queue = [...work]
+      await Promise.all(Array.from({ length: Math.max(1, Math.min(3, queue.length)) }, async () => {
+        for (let w = queue.shift(); w; w = queue.shift()) {
+          if (report.triageStop) break
+          const file = w.capture.files?.image || w.capture.files?.png
+          if (!file) continue
+          try {
+            const [lo, hi] = uncertain
+            const criteria = (w.node.criteria || []).filter((c) => c.check.type === 'semantic' && c.check.about !== 'console' && w.request?.questions?.[c.id])
+              .filter((c) => { const s = w.jev?.response?.answers?.[c.id]?.noul; return s != null && s > lo && s < hi })
+              .map((c) => ({ id: c.id, question: c.check.question }))
+            const context = { purpose: purposeOf(w.node, w.capture), url: w.capture.url, locale: w.capture.locale, viewport: w.entry.viewport, signals: { overflow: (w.capture.overflow || []).length, console_errors: (w.capture.console_errors || []).length } }
+            const { images } = await prepareImages({ page: await page(), file: join(runDir, file), cssWidth: w.capture.size?.width, full: true, settings: { max_width: settings.max_width, tile_height: settings.tile_height, max_tiles: settings.max_tiles } })
+            const tiles = images.slice(0, 4).map((i) => `data:${i.media_type};base64,${i.data}`)
+            report.triage.screens++
+            const lists = []; const results = []
+            const screen = await ask(w, { kind: 'screen', context, criteria, images: tiles }, {})
+            results.push(screen)
+            if (screen.findings) lists.push(screen.findings)
+            // the recording of this capture (same walk), as frames
+            const rec = entries.filter((e) => e.type === 'recording' && e.node === w.entry.node && e.locale === w.entry.locale && e.viewport === w.entry.viewport && e.captured_at && e.captured_at === w.capture.captured_at).pop()
+            if (rec && framesLeft > 0 && !report.triageStop) {
+              const side = await readJson(join(runDir, rec.file), null)
+              const video = side?.files?.webm || side?.files?.mp4
+              if (video) {
+                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.max_width })
+                framesLeft -= frames.length; report.triage.frames += frames.length
+                for (const batch of frameBatches(frames)) {
+                  const r = await ask(w, { kind: 'motion', context: { ...context, frames: batch.map((f) => ({ t_ms: f.t, why: f.why })) }, images: batch.map((f) => f.data) }, { frames: batch.map((f) => f.t) })
+                  results.push(r)
+                  if (r.findings) lists.push(r.findings)
+                }
+              }
+            }
+            const done = results.filter((r) => r.findings)
+            if (!done.length) { w.triage = { skipped: results[0]?.skipped || null, error: results[0]?.error || null }; continue }
+            const findings = mergeFindings([...lists, probeFindings(w.capture)])
+            report.triage.findings += findings.length; report.triage.shown += findings.filter((f) => f.shown).length
+            w.triage = { findings, criteria: screen.criteria || {}, model: done[0].model, via: done[0].via, cached: done.every((r) => r.cached), requests: done.length, ms: done.reduce((n, r) => n + (r.ms || 0), 0) || null }
+            // cascade (#71): criteria Clef is sure about skip the vision evaluator
+            if (settings.cascade) w.triageDecided = new Set(Object.entries(screen.criteria || {}).filter(([, p]) => p <= 0.15 || p >= 0.85).map(([id]) => id))
+          } catch (error) {
+            w.triage = { error: String(error.message || error).slice(0, 300) }
+          }
+        }
+      }))
+    },
+    async close () { if (browser) await browser.close().catch(() => {}) }
+  }
+}
+
 /**
  * Sends a JEV request through the organization's service (hosted evaluation) when a service token is set.
  * An allowance used up or a service without hosting stops further hosted calls for this run with one message.

package/src/lib/mail.mjs

@@ -1,5 +1,5 @@
 // E-mails of a flow, from wherever the project's stack puts them. Every source returns the same
-// message shape: { id, subject, from, to: [address], date, html, text }.
+// message shape: { id, subject, from, to: [address], cc: [address], date, html, text }.
 //   mailpit — built in (REST API), optional gate auth;
 //   script  — a project module for anything else (letter_opener, Mailosaur, IMAP, the app's own API);
 //   file    — `kaloko mail --file`: a message the agent obtained itself (.eml, .html, .json, .txt).
@@ -40,7 +40,7 @@ export function mailpit (mail) {
         hitId = hit.ID
       }
       const m = await get(`/api/v1/message/${encodeURIComponent(hitId)}`)
-      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
+      return { id: m.ID, subject: m.Subject || '', from: address(m.From), to: (m.To || []).map((t) => t.Address), cc: (m.Cc || []).map((t) => t.Address), date: m.Date, html: m.HTML || null, text: m.Text || null }
     }
   }
 }
@@ -95,6 +95,7 @@ export function normalize (m) {
     subject: m.subject ?? m.Subject ?? '',
     from: typeof from === 'object' && from ? address(from) : from || '',
     to: list(m.to ?? (m.To || []).map?.((t) => t.Address ?? t)),
+    cc: list(m.cc ?? (m.Cc || []).map?.((t) => t.Address ?? t)),
     date: m.date ?? m.Date ?? null,
     html: m.html ?? m.HTML ?? null,
     text: m.text ?? m.Text ?? null
@@ -109,8 +110,8 @@ export async function readMessageFile (file) {
   if (ext === '.eml' || ext === '.msg' || ext === '.mbox') return parseEml(raw)
   if (ext === '.json') return normalize(JSON.parse(raw.toString('utf8')))
   const body = raw.toString('utf8')
-  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], date: null, html: body, text: null }
-  return { id: '', subject: '', from: '', to: [], date: null, html: null, text: body }
+  if (ext === '.html' || ext === '.htm') return { id: '', subject: (body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || '').trim(), from: '', to: [], cc: [], date: null, html: body, text: null }
+  return { id: '', subject: '', from: '', to: [], cc: [], date: null, html: null, text: body }
 }
 
 // --- RFC 5322 / MIME, enough for what mailers send: multipart, base64, quoted-printable, charsets ---
@@ -175,6 +176,7 @@ export function parseEml (raw) {
     subject: h('subject'),
     from: h('from'),
     to: list(h('to')),
+    cc: list(h('cc')),
     date: top.headers.date ? new Date(top.headers.date).toISOString() : null,
     html: found.html ?? null,
     text: found.text ?? null

package/src/lib/manifest-core.mjs

@@ -63,10 +63,21 @@ export function slimCapture (cap) {
     ...(cap.design ? { design: cap.design } : {}), ...(cap.capture ? { capture: cap.capture } : {}),
     ...(cap.criteria_hash ? { criteria_hash: cap.criteria_hash } : {}),
     ...Object.fromEntries(DIM_FIELDS.filter((k) => cap[k]).map((k) => [k, cap[k]])),
-    ...(cap.ignore?.length ? { ignore: cap.ignore } : {}), ...(cap.unstable ? { unstable: cap.unstable } : {}), ...(cap.keyboard ? { keyboard: cap.keyboard } : {})
+    ...(cap.ignore?.length ? { ignore: cap.ignore } : {}), ...(cap.unstable ? { unstable: cap.unstable } : {}), ...(cap.keyboard ? { keyboard: cap.keyboard } : {}),
+    ...(cap.pii ? { pii: cap.pii } : {})
   }
 }
 
+/** Masked personal data of a capture as the canvas marks it (#64): mode, kinds found and up to 200 places. */
+export const piiOf = (p) => ({ mode: p.mode || 'redact', auto: !!p.auto, kinds: p.kinds || {}, rects: (p.rects || []).slice(0, 200) })
+
+/** The run's flag "personal data hidden" (#64): set when the run masks automatically or any capture was masked. */
+export function runPii (meta, nodes) {
+  const masked = nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((byVp) => Object.values(byVp || {}))).filter((c) => c?.pii)
+  if (!meta?.pii && !masked.length) return null
+  return { hidden: true, mode: meta?.pii?.mode || masked[0]?.pii?.mode || 'redact', auto: !!meta?.pii, captures: masked.length }
+}
+
 /** The keyboard walk of a capture as the canvas draws it: tab stops (numbered boxes) and the focus screenshots. */
 export const keyboardOf = (k) => k ? { complete: !!k.complete, total: k.total ?? (k.stops || []).length, stops: (k.stops || []).slice(0, 40), shots: (k.shots || []).slice(0, 10) } : null
 
@@ -100,12 +111,16 @@ export function assembleNode ({ node, pos, locales, viewportNames: allKeys, capt
       mail: cap.mail || null, eval: ev ? c.files.eval : null, ...(c.files.spec ? { spec: c.files.spec } : {}),
       // quick signals from the evaluator (probabilities: purpose met, expected language, looks broken)
       ...(ev?.signals ? { signals: ev.signals } : {}),
+      // what kind of problem Clef saw (#65, experimental): advisory findings, never a verdict
+      ...(ev?.findings?.length ? { findings: ev.findings.slice(0, 30) } : {}),
       ...(c.live ? { live: c.live } : {}),
       ...(cap.view?.revision ? { view: cap.view } : {}),
       // regions left out of pixel diffs (#20), regions that moved between repeated shots, the keyboard walk (#19)
       ...(cap.ignore?.length ? { ignore: cap.ignore.slice(0, 50) } : {}),
       ...(cap.unstable?.regions?.length ? { unstable: cap.unstable.regions.slice(0, 50) } : {}),
-      ...(cap.keyboard ? { keyboard: keyboardOf(cap.keyboard) } : {})
+      ...(cap.keyboard ? { keyboard: keyboardOf(cap.keyboard) } : {}),
+      // personal data hidden on this capture (#64): mode and the places the canvas marks
+      ...(cap.pii ? { pii: piiOf(cap.pii) } : {})
     }
     if (ev?.semantic || ev?.jev) semanticByKey[`${c.locale}|${c.viewport}`] = ev.semantic || ev.jev
     if (cap.metadata && !metadata[c.locale]) metadata[c.locale] = cap.metadata