kaloko 0.4.8 → 0.4.9

package/src/lib/build.mjs

@@ -29,9 +29,14 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
       const cap = await readJson(join(run.dir, c.file))
       const stored = await readJson(runPaths(run.dir).step(c.node, c.locale, c.viewport, 'eval.json'), null)
       const ev = stored?.source === evaluationSource(node, cap) ? stored : null
-      caps.push({ locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html || null, tree: cap.files.tree || null, eval: `steps/${c.node}/${c.locale}/${c.viewport}.eval.json` }, ...(cap.live ? { live: cap.live } : {}) })
+      // the step spec travels as its own file (Dev mode on screenshots, `kaloko spec`, drift on the service)
+      const spec = cap.spec ? `steps/${c.node}/${c.locale}/${c.viewport}.spec.json` : null
+      if (spec) await writeJson(join(run.dir, spec), cap.spec)
+      caps.push({ locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html || null, tree: cap.files.tree || null, eval: `steps/${c.node}/${c.locale}/${c.viewport}.eval.json`, spec }, ...(cap.live ? { live: cap.live } : {}) })
     }
-    nodes.push(assembleNode({ node, pos: pos.get(node.id), locales, viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id) }))
+    // focus screenshots of the keyboard walk (#19): only those of the current captures, not of earlier walks
+    const shots = new Set(caps.flatMap((c) => (c.cap.keyboard?.shots || []).map((s) => s.file)))
+    nodes.push(assembleNode({ node, pos: pos.get(node.id), locales, viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id && (a.kind !== 'focus' || shots.has(a.file))) }))
   }
   const summary = summarize(nodes, scenario.sessions)
 
@@ -126,19 +131,39 @@ async function buildStack ({ run, node, base, captures, attachments }) {
   return {
     ...base, status, by_locale, captures: {}, criteria, semantic: {}, metadata: {}, session: node.session || null,
     execution: node.execution ? { strategy: node.execution.strategy, instructions: node.execution.instructions } : { strategy: 'playwright', instructions: 'Open every page of the stack and capture it.' },
-    attachments: attachments.filter((a) => a.node === node.id).map((a) => ({ type: a.kind, title: a.title, file: a.file, locale: a.locale })),
+    attachments: attachments.filter((a) => a.node === node.id && a.kind !== 'focus').map((a) => ({ type: a.kind, title: a.title, file: a.file, locale: a.locale })),
     stack: { count: items.length, items: file, summary, fields, sample, semantic_sample: semanticSample, source: ['urls', 'file', 'sitemap', 'crawl', 'script'].find((k) => src[k]) || null, stale: items.filter((it) => it.stale).length }
   }
 }
 
+/**
+ * Lines for `kaloko build`: semantic criteria that failed or need a person, per step, locale and viewport, with the
+ * score and the one-sentence reason (#17). At most `max` lines.
+ */
+export function attentionLines (manifest, max = 20) {
+  const out = []
+  for (const n of manifest.nodes || []) {
+    for (const c of n.criteria || []) {
+      if (c.type !== 'semantic' || !['fail', 'needs_review'].includes(c.verdict)) continue
+      for (const [l, vps] of Object.entries(c.by_locale || {})) {
+        for (const [vp, r] of Object.entries(vps)) {
+          if (!['fail', 'needs_review'].includes(r.verdict)) continue
+          out.push(`  ${r.verdict === 'fail' ? '✕' : '?'} ${n.id} ${l}/${vp} ${c.id}${r.score != null ? ` ${Math.round(r.score * 100)} %${r.vision ? ' (with screenshot)' : ''}` : ''}${r.reason ? `: ${r.reason}` : r.note ? `: ${r.note}` : ''}`)
+        }
+      }
+    }
+  }
+  return out.length > max ? [...out.slice(0, max), `  … ${out.length - max} more on the canvas`] : out
+}
+
 export async function writePreview ({ run, manifest }) {
   const template = await readFile(join(packageRoot, 'viewer', 'index.html'), 'utf8')
   // the local preview opens from file://, where fetch cannot read files: stack indexes go inline
   // file:// cannot fetch: stack indexes and app UI trees travel inside the preview
   const withTrees = async (n) => {
-    if (!Object.values(n.captures || {}).some((vps) => Object.values(vps).some((c) => c?.tree))) return n
+    if (!Object.values(n.captures || {}).some((vps) => Object.values(vps).some((c) => c?.tree || c?.spec))) return n
     const captures = {}
-    for (const [l, vps] of Object.entries(n.captures)) { captures[l] = {}; for (const [vp, c] of Object.entries(vps)) captures[l][vp] = c?.tree ? { ...c, tree_inline: await readJson(join(run.dir, c.tree), null) } : c }
+    for (const [l, vps] of Object.entries(n.captures)) { captures[l] = {}; for (const [vp, c] of Object.entries(vps)) captures[l][vp] = c?.tree || c?.spec ? { ...c, ...(c.tree ? { tree_inline: await readJson(join(run.dir, c.tree), null) } : {}), ...(c.spec ? { spec_inline: await readJson(join(run.dir, c.spec), null) } : {}) } : c }
     return { ...n, captures }
   }
   // design runs: live steps open from the merged tree, where links between steps resolve (design.mjs copyLive)

package/src/lib/capture.mjs

@@ -2,6 +2,7 @@ import { writeFile, mkdir, rm } from 'node:fs/promises'
 import { dirname, relative, join } from 'node:path'
 import { buildHtmlView } from './htmlview.mjs'
 import { probe } from './probe.mjs'
+import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
 
@@ -37,6 +38,15 @@ export const toWebp = (page, png, opts = {}) => encodeImage(page, png, { ...opts
  * Capture options: scenario-wide `capture:` merged with the node's. Documentation runs use
  * { format: png, scale: 2, mask: [selectors], annotate: true }; daily acceptance keeps the WebP default.
  */
+/** The page spec (src/lib/devspec.mjs), kept under ~250 kB so it fits a design revision's capture record. */
+export async function stepSpec (page, tokens = null) {
+  try {
+    const spec = await page.evaluate(pageSpecScript(tokens))
+    while (spec.elements.length > 20 && JSON.stringify(spec).length > 250000) spec.elements = spec.elements.slice(0, Math.floor(spec.elements.length * 0.8))
+    return spec
+  } catch { return null }
+}
+
 export function captureOptions (scenario, node) {
   const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
   const q = Number(o.quality)
@@ -56,6 +66,8 @@ import { consoleEntriesFor } from './console.mjs'
 import { redact, redactDeep } from './secrets.mjs'
 import { readFields } from './stack.mjs'
 import { checkLinks, checkHreflang, requester, walkCache, NETWORK_ASSERTS, isolatedRequest, cacheScope } from './linkcheck.mjs'
+import { keyboardChecks, needsKeyboard, KEYBOARD_ASSERTS } from './keyboard.mjs'
+import { repeatRegions, resolveIgnore } from './stability.mjs'
 
 /** Probe arguments derived from the node's criteria. */
 export function probeArgs (node, maxChars, design = false) {
@@ -74,6 +86,9 @@ export function probeArgs (node, maxChars, design = false) {
 // md-menu, listboxes, tooltips). A step that leaves one open wants it on the capture.
 const OVERLAY = '[role=menu], [role=listbox], [role=tooltip], .MuiPopover-root, .MuiPopper-root, .MuiMenu-root, md-menu-content, .md-open-menu-container.md-active, .md-panel-outer-wrapper.md-panel-is-showing'
 
+const STACK_KEYBOARD = new Map()
+const STACK_KEYBOARD_PAGES = 5
+
 export async function settle (page) {
   try { await page.waitForLoadState('networkidle', { timeout: 5000 }) } catch {}
   try { await page.evaluate(() => document.fonts?.ready) } catch {}
@@ -95,7 +110,7 @@ export async function settle (page) {
  * Captures the current state of `page` for one step and locale in every
  * requested viewport: full-page PNG, rendered HTML, probe results.
  */
-export async function captureStep ({ page, node, locale, viewports, run, consoleFile, maxChars = 90000, since, extra = {}, format = null, options = null, designSummary = false, htmlView = null, item = null, fields = null }) {
+export async function captureStep ({ page, node, locale, viewports, run, consoleFile, maxChars = 90000, since, extra = {}, format = null, options = null, designSummary = false, htmlView = null, item = null, fields = null, tokens = null, stability = 0 }) {
   const opts = { ...(options || captureOptions(null, node)), ...(format ? { format } : {}) }
   // a page of a stack step lives under steps/<node>/items/<item>/…; its capture record names node and item
   const stepId = item ? `${node.id}/items/${item.id}` : node.id
@@ -132,6 +147,12 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         throw error
       }
     }
+    // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
+    if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens)
+    // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
+    const ignore = await resolveIgnore(page, node, locale, viewport)
+    // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
+    const unstable = stability > 1 ? await repeatRegions(page, { n: Math.min(6, stability), fullPage: opts.fullPage }).catch((error) => ({ error: String(error.message).slice(0, 200) })) : null
     // numbered marks on the highlighted elements, in criteria order (first box of each)
     const annotations = opts.annotate ? Object.entries(result.boxes || {}).filter(([, list]) => list?.length).map(([criterion, list], i) => ({ n: i + 1, criterion, x: list[0].x, y: list[0].y })) : []
     let image = paths.step(stepId, locale, viewport, opts.format)
@@ -168,6 +189,24 @@ export async function captureStep ({ page, node, locale, viewports, run, console
           : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
       }
     }
+    // keyboard and screen-reader checks (#19): a Tab walk and the accessibility tree, after the screenshot and after
+    // the console was read (focus handlers of the page must not count as its errors)
+    let keyboard = null
+    // stack steps walk the keyboard on their first few pages only (one template, hundreds of pages)
+    const kbKey = `${run.dir}|${node.id}|${viewport}`
+    const kbSampled = !item || (STACK_KEYBOARD.set(kbKey, (STACK_KEYBOARD.get(kbKey) || 0) + 1).get(kbKey) <= STACK_KEYBOARD_PAGES)
+    if (needsKeyboard(node) && !kbSampled) {
+      for (const c of node.criteria) if (KEYBOARD_ASSERTS.has(c.check.assert)) result.checks[c.id] = { ok: null, evidence: `keyboard walk runs on the first ${STACK_KEYBOARD_PAGES} pages of a stack step` }
+    }
+    if (needsKeyboard(node) && kbSampled) {
+      const kb = await keyboardChecks(page, node.criteria, {
+        saveShot: async (n, png) => { const f = paths.step(stepId, locale, viewport, `focus-${n}.png`); await writeFile(f, png); return relative(run.dir, f) }
+      })
+      Object.assign(result.checks, kb.checks)
+      result.boxes = { ...result.boxes, ...kb.boxes }
+      keyboard = kb.keyboard
+      for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: `Focus ${s.n}: ${s.name}`, file: s.file, at: new Date().toISOString() })
+    }
     // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
     let view = null
     if (htmlView) {
@@ -189,6 +228,9 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       ...extra,
       http_status: extra.http_status ?? result.http_status ?? null,
       capture: { format: String(image).split('.').pop(), scale: opts.scale, masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+      ...(ignore.length ? { ignore } : {}),
+      ...(unstable ? { unstable } : {}),
+      ...(keyboard ? { keyboard } : {}),
       ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}), ...(item.purpose ? { purpose: item.purpose } : {}) } } : {}),
       ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
       files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }

package/src/lib/design-cli.mjs

@@ -7,6 +7,7 @@ import { readJson, writeJson, UserError } from './util.mjs'
 import { readCaptures, addCapture, runPaths } from './run.mjs'
 import { latestCaptures, criteriaHash, evaluationSource } from './evaluate.mjs'
 import { captureStep } from './capture.mjs'
+import { loadTokens } from './drift.mjs'
 import { layout } from './scenario.mjs'
 import { launchEphemeral } from './browser.mjs'
 import { observeConsole } from './console.mjs'
@@ -45,6 +46,8 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
   const browser = await launchEphemeral(config)
   const paths = runPaths(run.dir)
   const mode = env.tokens_mode || ''
+  // the design folder's own set names the values in the step spec (the one its live pages reference)
+  const tokens = await loadTokens({ client: null, config, root, designDir: dir })
   let rendered = 0, reused = 0
   try {
     const page = await (await browser.newContext()).newPage()
@@ -58,7 +61,8 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
       for (const [vp, size] of Object.entries(viewports)) {
         const ck = captureKey(rev.id, vp, mode)
         const cached = noCache ? null : await fromCache({ root, output: config.output, key: ck })
-        if (cached && cached.record.criteria_hash === criteriaHash(node)) {
+        // a capture from before the step spec existed renders again, so Dev mode and `kaloko spec` have it
+        if (cached && cached.record.criteria_hash === criteriaHash(node) && cached.record.spec) {
           const record = await restoreCached({ cached, runDir: run.dir, paths, node: node.id, locale: rev.locale, viewport: vp })
           const file = paths.step(node.id, rev.locale, vp, 'capture.json')
           await writeJson(file, record)
@@ -69,7 +73,7 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
       if (Object.keys(todo).length) {
         const since = new Date().toISOString()
         await page.goto(`${server.url}/${rev.entry}`, { waitUntil: 'load' })
-        const results = await captureStep({ page, node, locale: rev.locale, viewports: todo, run, maxChars: config.evaluators?.jev?.max_state_chars, since, extra: { live }, designSummary: true })
+        const results = await captureStep({ page, node, locale: rev.locale, viewports: todo, run, maxChars: config.evaluators?.jev?.max_state_chars, since, extra: { live }, designSummary: true, tokens })
         for (const r of results) { await toCache({ root, output: config.output, key: captureKey(rev.id, r.viewport, mode), runDir: run.dir, record: r }); rendered++ }
       }
       say(`✓ ${node.id} ${rev.locale}  revision ${rev.id.slice(0, 8)}${Object.keys(todo).length ? '' : '  (unchanged, reused)'}`)
@@ -231,7 +235,7 @@ export async function syncDesign ({ config, root, run, scenario, env, client, ar
           out[k] = h
         }
         const stored = await readJson(runPaths(run.dir).step(rev.node, rev.locale, vp, 'eval.json'), null)
-        out.record = slimCapture(rec)
+        out.record = { ...slimCapture(rec), ...(rec.spec ? { spec: rec.spec } : {}) }
         out.evaluation = stored?.source === evaluationSource(node, rec) ? stored : null
         out.mode = env.tokens_mode || ''
         out.renderer = RENDERER