kaloko 0.21.0 → 0.21.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.21.0",
+  "version": "0.21.1",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/skills/kaloko/reference/integrations.md

@@ -102,9 +102,12 @@ kaloko schedule hook <id>       # URL + secret that start the run after a deploy
 - Findings: `GET …/scenarios/<id>/findings` (MCP `list_findings`) groups a
   scenario's open results by check, site, locale and step, with a status
   people set (acknowledged, ignored with a reason) and fixed when no page
-  fails. A scenario may open one Jira / Linear issue per new finding
-  (`PUT …/findings/sync { tracker, min_severity }`, Developer / Tester): at
-  most 20 per run, a comment on big changes, closed when fixed. Switching it
+  fails. A scenario may open one issue per new finding in Jira, Linear,
+  GitHub Issues or Plane (`PUT …/findings/sync { tracker, min_severity,
+  github?, plane?, routes? }`, Developer / Tester; routes send checks or
+  packs to another tracker): at most 20 per run, a comment on big changes,
+  closed when fixed. With `traffic:`, findings and issues carry clicks,
+  impressions and impact. Switching it
   on or ignoring findings is the team's call, not an agent's.
 - Does not work for: native apps, local or private addresses (localhost, IPs,
   `.local`), header credentials, stacks listed by crawl, script or file.

package/src/lib/evaluate.mjs

@@ -449,7 +449,7 @@ function triageContext ({ config, root, settings, report, fetchImpl, uncertain =
               const video = side?.files?.webm || side?.files?.mp4
               // a recording that cannot be read costs its frames only, never the screen's findings
               if (video) try {
-                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.frame_width || 640 })
+                const frames = await chooseFrames({ page: await page(), file: join(runDir, video), taps: (side.taps || []).map((t) => t.t ?? t), max: Math.min(settings.frames_per_step, framesLeft), maxW: settings.frame_width || 640, cacheDir })
                 framesLeft -= frames.length; report.triage.frames += frames.length
                 for (const batch of frameBatches(frames)) {
                   const r = await ask(w, { kind: 'motion', context: { ...context, frames: batch.map((f) => ({ t_ms: f.t, why: f.why })) }, images: batch.map((f) => f.data) }, { frames: batch.map((f) => f.t) })

package/src/lib/health-findings.mjs

@@ -8,6 +8,7 @@
 // Pure: no Node APIs. The shapes are documented in help/reference/health-api.md.
 import { parseHealthKey } from './health-index.mjs'
 import { severityRank, SEVERITIES } from './health-core.mjs'
+import { impactOf } from './traffic-core.mjs'
 
 export const HEALTH_FINDINGS_SCHEMA = 'kaloko.health-findings.v1'
 /** What a person may set; fixed is never set, it is derived. */
@@ -45,10 +46,13 @@ export function findingId (fp) {
 /**
  * Findings of a flow from its open results. rows: health_open rows ({ key, severity, url, first_run, first_at, misses
  * }); checks: { HC…: { text, pack } } of the newest run's index; edit: "step|page" → edit_url; seen: { run, at } of
- * the newest run (last seen of what it checked). Returns findings, worst first: { id, fingerprint, check, text, pack,
- * site, locale, step, severity, count, unchecked, samples: [{ page, url, edit_url }], first_seen, last_seen }.
+ * the newest run (last seen of what it checked); traffic: "step|page" → the page's Search Console and GA4 numbers
+ * (#157). Returns findings, worst first: { id, fingerprint, check, text, pack, site, locale, step, severity, count,
+ * unchecked, samples: [{ page, url, edit_url, traffic? }], first_seen, last_seen, traffic?, impact? }. With traffic,
+ * a finding adds up clicks and impressions of its pages that have numbers (`traffic.pages` of them) and its impact
+ * (the sum of each page's impact, traffic-core.mjs impactOf); without, it has neither.
  */
-export function groupFindings (rows, { checks = {}, edit = {}, seen = null } = {}) {
+export function groupFindings (rows, { checks = {}, edit = {}, seen = null, traffic = null } = {}) {
   const out = new Map()
   for (const r of rows || []) {
     const k = parseHealthKey(r.key)
@@ -64,9 +68,15 @@ export function groupFindings (rows, { checks = {}, edit = {}, seen = null } = {
     if (r.misses > 0) f.unchecked++
     else if (seen) f.last_seen = seen
     if (r.first_at && (!f.first_seen || r.first_at < f.first_seen.at)) f.first_seen = { run: r.first_run, at: r.first_at }
+    const ek = `${k.step}|${k.page}`
+    const t = traffic && Object.hasOwn(traffic, ek) && traffic[ek] && typeof traffic[ek] === 'object' ? traffic[ek] : null
+    if (t) {
+      const sum = (f.traffic ||= { clicks: 0, impressions: 0, pages: 0 })
+      sum.clicks += Number(t.clicks) || 0; sum.impressions += Number(t.impressions) || 0; sum.pages++
+      f.impact = (f.impact || 0) + impactOf(t, r.severity)
+    }
     if (f.samples.length < SAMPLE_MAX) {
-      const ek = `${k.step}|${k.page}`
-      f.samples.push({ page: k.page || null, url: r.url || null, edit_url: (edit && Object.hasOwn(edit, ek) ? edit[ek] : null) || null })
+      f.samples.push({ page: k.page || null, url: r.url || null, edit_url: (edit && Object.hasOwn(edit, ek) ? edit[ek] : null) || null, ...(t ? { traffic: { clicks: Number(t.clicks) || 0, impressions: Number(t.impressions) || 0 } } : {}) })
     }
   }
   return [...out.values()].sort(worstFirst)
@@ -79,8 +89,8 @@ export const FINDINGS_SUMMARY_SCHEMA = 'kaloko.health-findings-summary.v1'
  * the flow keeps open, plus the findings whose every page the run fixed (`fixed`: the diff's fixed results as rows;
  * count 0, last seen `previous`). An open finding counts the pages the run fixed in `fixed` too.
  */
-export function summarizeFindings (rows, { checks = {}, edit = {}, seen = null, fixed = [], previous = null } = {}) {
-  const list = groupFindings(rows, { checks, edit, seen })
+export function summarizeFindings (rows, { checks = {}, edit = {}, seen = null, fixed = [], previous = null, traffic = null } = {}) {
+  const list = groupFindings(rows, { checks, edit, seen, traffic })
   const at = new Map(list.map((f) => [f.id, f]))
   for (const g of groupFindings(fixed, { checks, edit })) {
     const f = at.get(g.id)
@@ -114,3 +124,54 @@ export const atLeastSeverity = (severity, min) => SEVERITIES.includes(min) && se
  * the issue last said.
  */
 export const countChanged = (before, now) => before != null && Math.abs(now - before) >= Math.max(3, Math.ceil(before * 0.25))
+
+// ---------- where issues go (#191) ----------
+
+/** Trackers the findings sync can open issues in. */
+export const SYNC_TRACKERS = ['jira', 'linear', 'github', 'plane']
+/** Routes a scenario may have; checks and packs per route. */
+export const ROUTE_MAX = 20
+const ROUTE_LIST_MAX = 50
+const CHECK_PATTERN = /^[A-Z]{1,6}[0-9]{0,6}\*?$/
+const PACK = /^[a-z][a-z0-9_-]{0,30}$/
+
+/**
+ * Routes of a sync setting from the API: [{ tracker, checks?: ['HC127', 'HC1*'], packs?: ['aeo'] }], in order. A check
+ * ends with * to take every check that starts so (HC1* is HC100 to HC199 and HC1000…). Throws an Error that says
+ * what is wrong; [] for none.
+ */
+export function cleanRoutes (list) {
+  if (list == null) return []
+  if (!Array.isArray(list)) throw new Error('routes is a list of { tracker, checks, packs }.')
+  if (list.length > ROUTE_MAX) throw new Error(`At most ${ROUTE_MAX} routes.`)
+  return list.map((r, i) => {
+    if (!r || typeof r !== 'object') throw new Error(`Route ${i + 1} is not an object.`)
+    const tracker = String(r.tracker || '')
+    if (!SYNC_TRACKERS.includes(tracker)) throw new Error(`Route ${i + 1}: tracker is ${SYNC_TRACKERS.join(', ')}.`)
+    const words = (v, name) => {
+      if (v == null) return []
+      if (!Array.isArray(v) || v.length > ROUTE_LIST_MAX) throw new Error(`Route ${i + 1}: ${name} is a list of at most ${ROUTE_LIST_MAX}.`)
+      return [...new Set(v.map((x) => String(x ?? '').trim()))]
+    }
+    const checks = words(r.checks, 'checks').map((c) => c.toUpperCase())
+    const packs = words(r.packs, 'packs').map((p) => p.toLowerCase())
+    const badCheck = checks.find((c) => !CHECK_PATTERN.test(c) || c === '*')
+    if (badCheck !== undefined) throw new Error(`Route ${i + 1}: ${badCheck.slice(0, 20) || '(empty)'} is not a check id (HC127) or a prefix (HC1*).`)
+    const badPack = packs.find((p) => !PACK.test(p))
+    if (badPack !== undefined) throw new Error(`Route ${i + 1}: ${badPack.slice(0, 20) || '(empty)'} is not a pack name (seo, aeo).`)
+    if (!checks.length && !packs.length) throw new Error(`Route ${i + 1}: name checks or packs.`)
+    return { tracker, ...(checks.length ? { checks } : {}), ...(packs.length ? { packs } : {}) }
+  })
+}
+
+const checkMatches = (pattern, check) => pattern.endsWith('*') ? String(check).startsWith(pattern.slice(0, -1)) : pattern === check
+/**
+ * The tracker a finding's issue goes to: the first route that names its check (or a prefix of it) or its pack, else
+ * the scenario's tracker.
+ */
+export function routeOf (f, routes = [], fallback = null) {
+  for (const r of routes || []) {
+    if ((r.checks || []).some((c) => checkMatches(c, f.check)) || (f.pack && (r.packs || []).includes(String(f.pack).toLowerCase()))) return r.tracker
+  }
+  return fallback
+}

package/src/lib/health-index.mjs

@@ -8,6 +8,7 @@
 // (worker/src/health-index.js); the shapes are documented in help/reference/health-api.md.
 import { healthOfNode, levelOf, findingSeverity, severityRank } from './health-core.mjs'
 import { worst } from './manifest-core.mjs'
+import { trafficView } from './traffic-core.mjs'
 
 export const HEALTH_INDEX_SCHEMA = 'kaloko.health-index.v1'
 export const HEALTH_AGGREGATE_SCHEMA = 'kaloko.health-aggregate.v1'
@@ -42,14 +43,16 @@ const firstUrl = (vps) => { for (const c of Object.values(vps || {})) if (c?.url
  * [{ locale, step, check, counts, top: [{ page, path, url, verdict, evidence }] }], open: [[key, severity, url]],
  * open_total, truncated, passed, edit }. passed says which checks passed on which page: { "step|locale": { sigs:
  * ["HC103,HC112", …], pages: { <page>: <index into sigs> } } }; only a result that passed counts as fixed later
- * (passedIn). edit maps "step|page" to the edit_url of a stack page with open results (findings link it).
+ * (passedIn). edit maps "step|page" to the edit_url of a stack page with open results (findings link it); traffic
+ * maps "step|page" to the Search Console and GA4 numbers of such a page when its stack has `traffic:` (#157), for the
+ * same pages and bound (findings add them up, an issue shows them).
  */
 export function buildHealthIndex (manifest, stacks = {}) {
   const checks = Object.create(null); const groups = new Map(); const open = []; const pages = new Set(); const ok = new Map()
   // where a page with open results is edited (a stack item's edit_url): "step|page" → URL, kept below for the open list only
-  const edits = new Map()
+  const edits = new Map(); const visits = new Map()
   const totals = { results: 0, ...blank() }
-  const add = (step, locale, page, path, url, check, verdict, evidence, editUrl = null) => {
+  const add = (step, locale, page, path, url, check, verdict, evidence, editUrl = null, traffic = null) => {
     if (!VERDICTS.includes(verdict)) return
     pages.add(`${step}|${page}`)
     const gk = `${locale}\u0000${step}\u0000${check}`
@@ -63,6 +66,7 @@ export function buildHealthIndex (manifest, stacks = {}) {
     if (list.length < TOP_KEEP) list.push({ page, ...(path ? { path } : {}), url: url || null, verdict, ...(evidence ? { evidence: cut(evidence) } : {}) })
     open.push([healthKey(step, locale, check, page), severity, url || null])
     if (editUrl && typeof editUrl === 'string' && /^https?:\/\//i.test(editUrl)) edits.set(`${step}|${page}`, cut(editUrl, 500))
+    if (traffic && typeof traffic === 'object') { const t = trafficView(traffic); if (t && Object.keys(t).length) visits.set(`${step}|${page}`, t) }
   }
   for (const n of manifest?.nodes || []) {
     if (n.kind === 'decision') continue
@@ -85,7 +89,7 @@ export function buildHealthIndex (manifest, stacks = {}) {
             if (!was || worst([was.v, x[0]]) !== was.v) per.set(k, { locale, id, v: x[0], e: x[1] })
           }
         }
-        for (const { locale, id, v, e } of per.values()) add(n.id, locale, String(it.id), it.path || null, it.url || firstUrl(it.captures?.[locale]), id, v, e, it.edit_url)
+        for (const { locale, id, v, e } of per.values()) add(n.id, locale, String(it.id), it.path || null, it.url || firstUrl(it.captures?.[locale]), id, v, e, it.edit_url, it.traffic)
       }
       continue
     }
@@ -115,16 +119,18 @@ export function buildHealthIndex (manifest, stacks = {}) {
   // edit links of the pages in the kept open list only, at most EDIT_MAX of them (worst first, like the list)
   const kept = open.slice(0, OPEN_MAX)
   const edit = Object.create(null); let links = 0
+  const traffic = Object.create(null); let numbers = 0
   for (const [key] of kept) {
-    if (links >= EDIT_MAX) break
+    if (links >= EDIT_MAX && numbers >= EDIT_MAX) break
     const { step, page } = parseHealthKey(key)
     const k = `${step}|${page}`
-    if (edits.has(k) && !Object.hasOwn(edit, k)) { edit[k] = edits.get(k); links++ }
+    if (links < EDIT_MAX && edits.has(k) && !Object.hasOwn(edit, k)) { edit[k] = edits.get(k); links++ }
+    if (numbers < EDIT_MAX && visits.has(k) && !Object.hasOwn(traffic, k)) { traffic[k] = visits.get(k); numbers++ }
   }
   return {
     schema: HEALTH_INDEX_SCHEMA, checks, totals: { ...totals, pages: pages.size },
     groups: [...groups.values()].map(({ fails, unsure, ...g }) => ({ ...g, top: [...fails, ...unsure].slice(0, TOP_KEEP) })),
-    open: kept, open_total: open.length, truncated: open.length > OPEN_MAX, passed, edit
+    open: kept, open_total: open.length, truncated: open.length > OPEN_MAX, passed, edit, ...(numbers ? { traffic } : {})
   }
 }
 

package/src/lib/triage.mjs

@@ -4,6 +4,7 @@
 // CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID straight to Workers AI. Findings never change a verdict.
 import { join } from 'node:path'
 import { readFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
 import { readJson, writeJson, sha256 } from './util.mjs'
 import { buildTriageRequest, parseTriage, mergeFindings, clefModelId, TRIAGE_PROMPT_VERSION, MAX_TRIAGE_IMAGES } from './triage-core.mjs'
 
@@ -77,15 +78,24 @@ export function hostedTriageSender (config, report) {
   return send
 }
 
+// bump when the choice of frames or their encoding changes, so cached frames are chosen again
+const FRAMES_VERSION = 1
+
 /**
  * Frames of a recording for the motion questions, chosen in a browser page: around each tap, where the picture changes
  * most between samples (a jump, a flash, a message), and the last frame. Returns [{ t, why, data }] in time order,
- * at most `max` frames, each a WebP data URL `maxW` wide.
+ * at most `max` frames, each a WebP data URL `maxW` wide. With `cacheDir` the frames are kept by the recording's
+ * content and the settings, so evaluating the same recording again sends the very same frames (and its triage answers
+ * from the cache) even when the browser decodes or encodes a frame differently the second time.
  */
-export async function chooseFrames ({ page, file, taps = [], max = 6, maxW = 1024 }) {
+export async function chooseFrames ({ page, file, taps = [], max = 6, maxW = 1024, cacheDir = null }) {
   if (!(max > 0)) return []
-  const src = `data:video/${file.endsWith('.mp4') ? 'mp4' : 'webm'};base64,${(await readFile(file)).toString('base64')}`
-  return page.evaluate(async ({ src, taps, max, maxW }) => {
+  const bytes = await readFile(file)
+  const cacheFile = cacheDir && join(cacheDir, `frames-${createHash('sha256').update(bytes).update(JSON.stringify({ taps, max, maxW, v: FRAMES_VERSION })).digest('hex')}.json`)
+  const kept = cacheFile && await readJson(cacheFile, null)
+  if (Array.isArray(kept)) return kept
+  const src = `data:video/${file.endsWith('.mp4') ? 'mp4' : 'webm'};base64,${bytes.toString('base64')}`
+  const frames = await page.evaluate(async ({ src, taps, max, maxW }) => {
     const v = document.createElement('video'); v.muted = true; v.preload = 'auto'; v.src = src
     // every wait has a limit: a stalled video must not hang the evaluation
     const within = (p, ms) => Promise.race([p, new Promise((ok, no) => setTimeout(() => no(new Error('the recording did not answer in time')), ms))])
@@ -96,13 +106,25 @@ export async function chooseFrames ({ page, file, taps = [], max = 6, maxW = 102
     const dur = Number.isFinite(v.duration) ? v.duration : 0
     if (!dur) return []
     const seek = (t) => within(new Promise((ok) => { v.onseeked = ok; v.currentTime = Math.min(Math.max(0, t), Math.max(0, dur - 0.01)) }), 5000)
+    // `seeked` (even at readyState 4) does not mean the frame can be painted yet: on a busy machine the first draw
+    // sometimes paints nothing, which read as a change from an empty picture and moved the chosen frames. A decoded
+    // frame is opaque, so the draw is repeated until its middle row holds one.
+    const draw = async (g, w, h) => {
+      const painted = () => { const row = g.getImageData(0, h >> 1, w, 1).data; for (let i = 3; i < row.length; i += 4) if (row[i]) return true; return false }
+      for (let tries = 0; ; tries++) {
+        g.clearRect(0, 0, w, h); g.drawImage(v, 0, 0, w, h)
+        if (painted()) return
+        if (tries >= 100) throw new Error('a frame of the recording could not be drawn')
+        await new Promise((ok) => setTimeout(ok, 20))
+      }
+    }
     const small = document.createElement('canvas'); small.width = 64; small.height = Math.max(1, Math.round(64 * v.videoHeight / v.videoWidth))
     const sg = small.getContext('2d', { willReadFrequently: true })
     // sampled differences: 4 samples a second, at most 240
     const step = Math.max(0.25, dur / 240)
     const diffs = []; let prev = null
     for (let t = 0; t <= dur; t += step) {
-      await seek(t); sg.drawImage(v, 0, 0, small.width, small.height)
+      await seek(t); await draw(sg, small.width, small.height)
       const px = sg.getImageData(0, 0, small.width, small.height).data
       if (prev) { let d = 0; for (let i = 0; i < px.length; i += 4) d += Math.abs(px[i] - prev[i]) + Math.abs(px[i + 1] - prev[i + 1]) + Math.abs(px[i + 2] - prev[i + 2]); diffs.push({ t, d: d / (px.length / 4) / 765 }) }
       prev = px
@@ -117,11 +139,13 @@ export async function chooseFrames ({ page, file, taps = [], max = 6, maxW = 102
     const list = chosen.length > max ? [...chosen.slice(0, max - 1), chosen.at(-1)] : chosen
     const big = document.createElement('canvas'); const s = Math.min(1, maxW / v.videoWidth)
     big.width = Math.round(v.videoWidth * s); big.height = Math.round(v.videoHeight * s)
-    const bg = big.getContext('2d')
+    const bg = big.getContext('2d', { willReadFrequently: true })
     const out = []
-    for (const f of list) { await seek(f.t); bg.drawImage(v, 0, 0, big.width, big.height); out.push({ t: Math.round(f.t * 1000), why: f.why, data: big.toDataURL('image/webp', 0.8) }) }
+    for (const f of list) { await seek(f.t); await draw(bg, big.width, big.height); out.push({ t: Math.round(f.t * 1000), why: f.why, data: big.toDataURL('image/webp', 0.8) }) }
     return out
   }, { src, taps, max, maxW })
+  if (cacheFile && frames.length) await writeJson(cacheFile, frames)
+  return frames
 }
 
 /** Frames in requests of MAX_TRIAGE_IMAGES, each overlapping the previous by one frame so a change at the seam is seen. */