kaloko 0.4.7 → 0.4.9

package/skills/kaloko/reference/mcp.md

@@ -23,13 +23,18 @@ file.
 | Tool | Use it for |
 |---|---|
 | `list_runs` | published runs, newest first; filters `scenario`, `project`, `verdict`, `author`, `environment`, `branch`, `limit` |
-| `get_run_summary` | one run: statuses per step and locale, criteria that need attention with evidence, screenshot URLs, verdict, baseline, canvas URL, open feedback |
+| `get_run_summary` | one run: statuses per step and locale, criteria that need attention with evidence (semantic ones with score and the evaluator's one-sentence `reason` per locale and viewport), regions ignored in pixel diffs and regions seen moving (`ignored_regions`, `unstable_regions`), screenshot URLs, verdict, baseline, canvas URL, open feedback |
 | `get_feedback` | run verdict with note, approved steps, unresolved rejections and comments, failed manual criteria (`run_id`, or `scenario` for its latest run) |
 | `list_notifications` | the token owner's inbox (`unread_only`, `limit`) |
 | `list_projects` | projects of the organization with counts; the slug is the `project` filter of `list_runs` and the value for `project:` in a scenario |
 | `get_criteria_verdicts` | reviewer verdicts on manual or overridden criteria of a run |
+| `get_review_queue` | what a person still has to decide in a run (`run_id`, or `scenario` / nothing for the newest run): steps in review order with reasons, decisions, approvals carried over unchanged steps with their origin, counts and `review_url` to give the person |
+| `decide_step` | records your (the agent's) decision on a step: `approve`, or `return` with a `note`; marked as an agent's and not counted as a person's. Never use it to approve for a person |
+| `revoke_carried_approval` | revokes an approval carried over to a step that looks unchanged but should be looked at again (`run_id`, `step`) |
 | `get_step_assets` | documentation layer (Business): newest capture of every step of a scenario with stable image, HTML and metadata URLs (`scenario`, `step`, `locale`, `viewport`, `accepted`) |
 | `list_step_changes` | documentation layer: steps whose capture or status changed since the previous accepted run or `since` (`scenario`, `since`, `accepted`); see [documentation.md](documentation.md) |
+| `get_started` | where the organization stands (`empty`, `sample`, `active`), its sample run, the next steps and a prompt; call it in a new project, see [first-run.md](first-run.md) |
+| `add_sample_run` | adds the sample run (or returns the one there is); needs an upload or full token and the Developer / Tester role |
 
 ## Comment tools
 
@@ -60,6 +65,8 @@ The same data and permissions as the REST API; each tool is its REST route.
 |---|---|
 | `get_draft` | which revision each step shows, step status, who changed it (person or agent) |
 | `get_step` | the handoff: live files, signed entry URL, token set, page structure, open comments (`ref`: draft, design, vN, run id) |
+| `get_step_spec` | what to build for a step as approved: elements with role, name and selector, styles mapped to tokens, states, contrast (`ref`: design by default, draft, vN, run id; `format` text or json) |
+| `get_step_drift` | how an implementation run (`run`: id or `latest`) differs from the design reference, per element: expected token against the actual value |
 | `get_step_history` | versions, and revisions of a step with authors |
 | `list_step_changes` | steps that differ between two references (default design reference → draft) |
 | `get_references`, `get_tokens` | design reference and baseline; the project's token set |

package/src/cli.mjs

@@ -1,6 +1,7 @@
 import { writeFile, access, readFile, mkdir } from 'node:fs/promises'
 import { existsSync } from 'node:fs'
 import { gzipSync } from 'node:zlib'
+import { createHash } from 'node:crypto'
 import { join, resolve, relative, dirname } from 'node:path'
 import { pathToFileURL } from 'node:url'
 import yaml from 'js-yaml'
@@ -16,7 +17,8 @@ import { newContext, accountFor, gateHeaders, gateOrigins, originMatcher, gateFo
 import { configureGate } from './lib/linkcheck.mjs'
 import { rememberEnvironment } from './lib/secrets.mjs'
 import { evaluateRun, semanticPrefetch } from './lib/evaluate.mjs'
-import { buildManifest, writePreview } from './lib/build.mjs'
+import { buildManifest, writePreview, attentionLines } from './lib/build.mjs'
+import { stabilityCmd } from './lib/stability.mjs'
 import { localized, readJson, UserError, globToRegExp, failureStreak, expiryNote } from './lib/util.mjs'
 import { packageRoot } from './lib/schema.mjs'
 import { compareRuns, formatComparison } from './lib/compare.mjs'
@@ -29,9 +31,12 @@ import { resolveItems, fieldsOf, DEFAULT_LIMIT } from './lib/stack.mjs'
 import { DEFAULTS as JEV_DEFAULTS } from './lib/jev.mjs'
 import { expandPacks } from './lib/packs.mjs'
 import { upsertPrComment } from './lib/pr.mjs'
+import { formatReviewQueue, shareReviewLine } from './lib/review.mjs'
 import { htmlViewAllowed } from './lib/htmlview.mjs'
 import { service, serviceUrl } from './lib/service.mjs'
 import { login } from './lib/login.mjs'
+import { initNextSteps, loginNextSteps } from './lib/onboarding.mjs'
+import { specCmd, driftCmd } from './lib/spec-cli.mjs'
 import { walkDesign, syncDesign, publishVersion, historyCmd, referenceCmd, tokensCmd, exportLive, localRevisions, branchCmd, uploadBlobs } from './lib/design-cli.mjs'
 import { designDir, readBase, headKey } from './lib/design.mjs'
 import { lintDesign, formatLint } from './lib/design-lint.mjs'
@@ -98,6 +103,8 @@ Run
                                     verdicts and what failed (--no-evaluate skips that; kaloko evaluate redoes it)
        [--items a,b] [--limit N]    Stack steps (kind: stack, many pages of one template): only pages whose id or
                                     path contains a, b; at most N pages (default items.limit, 500)
+       [--stability N]              Take N screenshots of each capture a moment apart and note regions that move
+                                    (clocks, carousels); kaloko stability turns them into ignore suggestions
   status                            Show what the current run has captured
                                     Scenarios may declare "sessions" (people in the flow); walk keeps one browser context per session and locale
 
@@ -111,7 +118,11 @@ Results
   share [--expires 30] [--pr]       Upload the run to kaloko.app (KALOKO_TOKEN, tester role);
                                     --pr adds or updates one comment on the branch's pull request via gh
   feedback [--run id] [--scenario id] [--json]
-                                    Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies
+                                    Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies;
+                                    returned steps first, with their notes
+  review [--run id] [--scenario id] [--json]
+                                    What a person still has to decide on a shared run (failed, unsure, changed, returned,
+                                    commented steps), approvals carried over unchanged steps, and the review-mode link
   comment [--step id | --flow] --note "…" [--status rejected|approved] [--run id] [--scenario id]
   comment --reply|--resolve|--reopen|--edit|--delete <id> [--note "…"] [--flow]
                                     Comment, answer and close feedback from the CLI (resolve with "Fixed in <commit>")
@@ -136,6 +147,10 @@ Results
                                     Newest capture of every step as steps/<step>/<locale>/<viewport>.<ext> plus
                                     index.json (documentation layer, Business plan); unchanged files are skipped
   public [--run id] [--off]         Make a shared run readable without sign-in (admin; testers may refresh an already public scenario)
+  stability [--last 4 | --runs a,b] [--same-commit] [--apply] [--json]
+                                    Regions that change on every run or move while the page is idle (clocks, ads,
+                                    carousels): ignore suggestions per step in <run>/stability.yml; --apply adds
+                                    them to the scenario (ignore:), and pixel diffs then skip them
   compare [--run id] [--baseline id | --design] [--json]
                                     Text diff of two runs: step statuses, criterion verdicts, scores and evidence
                                     (default: current run against the scenario baseline, else the previous run)
@@ -153,6 +168,12 @@ Design (environments with kind: design; live HTML steps in a folder)
   share [--name <text>] [--branch b]  Publish the draft as a version of the flow (a run): "v4 · 3 changed, 17 unchanged"
   history [--scenario id] [--step id] [--locale l] [--json]
                                     Versions of the flow; revisions of a step with who (person or agent) and when
+  spec [step] [--ref design|draft|vN|<run id>] [--locale l] [--viewport v] [--local] [--json]
+                                    What to build for a step as approved: elements with role, name and selector, styles
+                                    mapped to DTCG tokens (or the nearest one), :hover/:focus/:active/:disabled, contrast
+  drift [step] [--ref design|vN] [--remote <run id|latest>] [--json]
+                                    How this implementation run differs from the approved design (expected token vs actual
+                                    value, raw values, missing elements and states); exit code 1 when something differs
   reference [--scenario id] (--design|--baseline) (--run id|--version n|--off) [--branch b]
                                     Show or set the design reference (accepted version) and the baseline
   tokens pull|push|check|validate [--project p] [--file tokens.json|tokens.resolver.json] [--mode m] [--json]
@@ -563,7 +584,13 @@ const commands = {
       await cp(join(packageRoot, 'templates', 'steps', 'consent.mjs'), consentStep)
       say('Added qa/flows/steps/consent.mjs (accepts the consent dialog, if the site has one)')
     }
-    say(`Next: run kaloko login to approve a token in the browser (or create one in the app under Settings → API tokens and put KALOKO_TOKEN into .env).\nScenario template: ${join(packageRoot, 'templates', 'scenario.yml')}`)
+    // the organization's sample run and the way to a first real flow (src/lib/onboarding.mjs)
+    let written = ''
+    try { written = await readFile(existsSync(file) ? file : join(cwd, LEGACY_CONFIG_FILE), 'utf8') } catch {}
+    const org = typeof args.org === 'string' ? args.org : (written.match(/^org:\s*["']?([a-z0-9-]+)["']?\s*$/m) || [])[1]
+    const base = serviceUrl({ share: { url: (written.match(/^share:[ \t]*\n(?:[ \t]+.*\n)*?[ \t]+url:[ \t]*["']?(https?:\/\/[^\s"']+)/m) || [])[1] } })
+    say(initNextSteps({ base, org }).join('\n'))
+    say(`Scenario template: ${join(packageRoot, 'templates', 'scenario.yml')}`)
   },
 
   async validate (args) {
@@ -691,7 +718,7 @@ const commands = {
         return
       }
       const since = (await readCaptures(run.dir)).filter((c) => c.node === node.id).map((c) => c.at).pop()
-      const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null })
+      const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, stability: Number(args.stability) || 0 })
       for (const r of results) {
         const failed = Object.entries(r.checks).filter(([, v]) => v.ok === false).map(([k]) => k)
         say(`✓ ${node.id} ${locale} ${r.viewport}  ${r.size.width}×${r.size.height}px  ${failed.length ? `failing checks: ${failed.join(', ')}` : 'checks ok'}${r.blocked_requests.length ? `  blocked: ${r.blocked_requests.length}` : ''}`)
@@ -815,7 +842,7 @@ const commands = {
               // the automatic capture runs unless this script invocation captured the step itself: a crop is an
               // attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
               let captured = false
-              const capture = (opts = {}) => { captured = true; return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile: conn && !sess ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {} }).then((r) => { prefetch?.add(r, run.dir); return r }) }
+              const capture = (opts = {}) => { captured = true; return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile: conn && !sess ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); return r }) }
               await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
               if (!captured) await capture()
               say(`✓ ${node.id} ${locale}`)
@@ -936,6 +963,7 @@ const commands = {
     const report = await evaluateRun({ run, scenario, config, root })
     const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
     say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
+    if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
     if (report.skipped.length) {
       // one line per reason: "12 captures: TYPESAFE_API_KEY is not set …"
       const byReason = new Map()
@@ -951,9 +979,15 @@ const commands = {
     const file = await writePreview({ run, manifest })
     const s = manifest.summary
     say(`${manifest.id}: ${s.pass} pass · ${s.partial} partial · ${s.fail} fail · ${s.skip} not checked\nPreview: ${file}`)
+    const why = attentionLines(manifest); if (why.length) say(`AI verdicts to act on (score and why):\n${why.join('\n')}`)
     return file
   },
 
+  async stability (args) {
+    const { config, root, run, scenario } = await runContext(args)
+    await stabilityCmd({ root, config, run, scenario, args, say })
+  },
+
   async preview (args) {
     const file = await commands.build(args)
     say(pathToFileURL(file).href)
@@ -976,7 +1010,7 @@ const commands = {
     await writePreview({ run, manifest })
     const files = new Set()
     for (const n of manifest.nodes) {
-      for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree].filter(Boolean).forEach((p) => files.add(p))
+      for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree, c.spec].filter(Boolean).forEach((p) => files.add(p))
       for (const a of n.attachments || []) files.add(a.file)
       // stack steps: the items index and every page's files
       if (n.stack?.items) {
@@ -1000,9 +1034,10 @@ const commands = {
     const body = async (p) => { const raw = await readFile(join(run.dir, p)); const gz = /\.(html|json)$/.test(p) && raw.length > 512; return { data: gz ? gzipSync(raw, { level: 6 }) : raw, gz } }
     // the file list goes first: the service checks limits once and each upload is then a plain write
     // (an older service without the plan answers 404/405 and every upload checks itself, as before)
-    if (files.size > 20) {
+    // screenshots name their SHA-256: the service checks it on arrival and compares unchanged steps with it
+    if (files.size) {
       const plan = []
-      for (const p of files) plan.push({ path: p, bytes: (await body(p)).data.length })
+      for (const p of files) { const { data } = await body(p); plan.push({ path: p, bytes: data.length, ...(/\.(png|webp|jpe?g)$/i.test(p) ? { sha256: createHash('sha256').update(data).digest('hex') } : {}) }) }
       await client.request('POST', `/orgs/${config.org}/runs/${manifest.id}/files`, { files: plan }).catch((e) => { if (!/HTTP (404|405)/.test(e.message)) throw e })
     }
     const queue = [...files]
@@ -1036,6 +1071,9 @@ const commands = {
     meta.shared = { url: fin.url, at: new Date().toISOString(), expires_days: created.expires_days }
     await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
     say(`Shared ${manifest.id} (${files.size} files, expires in ${created.expires_days} days)\n${fin.url}`)
+    // what waits for a person, with the review-mode link to give them (approvals of unchanged steps carry over)
+    const review = shareReviewLine(fin.review)
+    if (review) say(review)
     const capped = expiryNote(days, created.expires_days)
     if (capped) say(`Note: ${capped}`)
     if (args.pr) {
@@ -1080,6 +1118,25 @@ const commands = {
     await historyCmd({ config, scenarioId, client: service(config), args, say })
   },
 
+  async spec (args) {
+    const { config, root } = await context(args)
+    let run = null
+    try { run = await resolveRun({ root, output: config.output, run: args.run }) } catch {}
+    const scenario = run ? await loadScenario(resolve(root, run.meta.scenario_file)).catch(() => null) : typeof args.scenario === 'string' && /\.ya?ml$/.test(args.scenario) ? await loadScenario(resolve(args.scenario)) : null
+    const scenarioId = await scenarioIdOf(args, config, root)
+    let client = null
+    if (!args.local) { try { client = service(config) } catch (e) { if (!run) throw e } }
+    await specCmd({ config, client, run, scenario, scenarioId, args, say })
+  },
+
+  async drift (args) {
+    const { config, root } = await context(args)
+    let run = null
+    try { run = await resolveRun({ root, output: config.output, run: args.run }) } catch {}
+    const scenarioId = await scenarioIdOf(args, config, root)
+    await driftCmd({ config, client: service(config), run, scenarioId, args, say })
+  },
+
   async reference (args) {
     const { config, root } = await context(args)
     const scenarioId = await scenarioIdOf(args, config, root)
@@ -1132,12 +1189,38 @@ const commands = {
       for (const x of r.replies || []) say(`      ↳ ${x.email}${x.agent ? ` via ${x.agent}` : ''}: ${x.deleted ? '(deleted)' : x.note}`)
     }
     const flow = data.flow_comments || []
+    // how many steps still wait for a person (review mode); an older service answers 404 and the line is left out
+    const queue = await client.request('GET', `/orgs/${config.org}/review-queue?run=${encodeURIComponent(data.run.id)}`).catch(() => null)
+    if (queue?.counts) say(`  review: ${queue.counts.pending} of ${queue.counts.queue} step(s) wait for a person${queue.counts.carried ? `, ${queue.counts.carried} unchanged kept their approval` : ''} (kaloko review; review mode: ${queue.review_url})`)
     if (!data.items.length && !flow.length) { say('  No open rejections or comments.'); return }
-    for (const r of data.items) thread(r, r.node_id ? `step ${r.node_id}` : 'whole run')
+    // returned steps first: a reviewer said what should change
+    const returned = data.items.filter((r) => r.status === 'rejected')
+    if (returned.length) say(`Returned steps (${returned.length}), fix these first:`)
+    for (const r of returned) thread(r, r.node_id ? `step ${r.node_id}` : 'whole run')
+    if (returned.length && data.items.length > returned.length) say('Comments:')
+    for (const r of data.items.filter((x) => x.status !== 'rejected')) thread(r, r.node_id ? `step ${r.node_id}` : 'whole run')
     for (const r of flow) thread(r, 'whole flow')
     say(`\nAnswer with: kaloko comment --reply <id> --note "…"; close with: kaloko comment --resolve <id> --note "Fixed in <commit>"${flow.length ? ' (add --flow for flow comments)' : ''}`)
   },
 
+  /**
+   * What a person still has to decide on a shared run: the service's review queue (failed, unsure, changed since the
+   * accepted run, returned, commented), the approvals carried over unchanged steps, and the review-mode link. For
+   * agents: report it to the person; never approve on their behalf.
+   */
+  async review (args) {
+    const { config, root } = await context(args)
+    const client = service(config)
+    let runId = typeof args.run === 'string' ? args.run : null
+    let scenarioId = typeof args.scenario === 'string' ? args.scenario : null
+    if (!runId && !scenarioId) {
+      try { const r = await resolveRun({ root, output: config.output }); if (r.meta.shared) runId = r.meta.id; else scenarioId = r.meta.scenario.id } catch {}
+    }
+    const q = new URLSearchParams({ run: runId || 'latest', ...(scenarioId ? { scenario: scenarioId } : {}) })
+    const data = await client.request('GET', `/orgs/${config.org}/review-queue?${q}`)
+    say(args.json ? JSON.stringify(data, null, 2) : formatReviewQueue(data))
+  },
+
   /**
    * Comments from the CLI: on a step (--step), the whole run (default) or the whole flow (--flow); reply, resolve
    * (with a closing note), reopen, edit and delete by id. Targets the shared run (--run, default the current one).
@@ -1236,6 +1319,7 @@ const commands = {
     if (args.print) return
     say(`✓ Signed in to ${r.org_name || r.org} as ${r.email} (${r.scope}; expires ${String(r.expires_at || '').slice(0, 10)}). ${tokenEnv} saved to ${r.file}.`)
     if (config.org && r.org && config.org !== r.org) say(`! ${CONFIG_FILE} names the organization ${config.org}; the token is for ${r.org}.`)
+    for (const line of loginNextSteps({ base, org: r.org })) say(line)
     let projectEnv = ''
     try { projectEnv = await readFile(join(process.cwd(), '.env'), 'utf8') } catch {}
     if (inShell || new RegExp(`^\\s*(export\\s+)?${tokenEnv}\\s*=`, 'm').test(projectEnv)) say(`! ${tokenEnv} is also set ${inShell ? 'in your shell' : 'in this project\'s .env'}, which wins over ~/.config/kaloko/.env. Remove it there to use the new token.`)

package/src/lib/build.mjs

@@ -29,9 +29,14 @@ export async function buildManifest ({ run, scenario, config, root, catalog = []
       const cap = await readJson(join(run.dir, c.file))
       const stored = await readJson(runPaths(run.dir).step(c.node, c.locale, c.viewport, 'eval.json'), null)
       const ev = stored?.source === evaluationSource(node, cap) ? stored : null
-      caps.push({ locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html || null, tree: cap.files.tree || null, eval: `steps/${c.node}/${c.locale}/${c.viewport}.eval.json` }, ...(cap.live ? { live: cap.live } : {}) })
+      // the step spec travels as its own file (Dev mode on screenshots, `kaloko spec`, drift on the service)
+      const spec = cap.spec ? `steps/${c.node}/${c.locale}/${c.viewport}.spec.json` : null
+      if (spec) await writeJson(join(run.dir, spec), cap.spec)
+      caps.push({ locale: c.locale, viewport: c.viewport, cap, ev, files: { png: cap.files.png, thumb: cap.files.thumb || null, html: cap.files.html || null, tree: cap.files.tree || null, eval: `steps/${c.node}/${c.locale}/${c.viewport}.eval.json`, spec }, ...(cap.live ? { live: cap.live } : {}) })
     }
-    nodes.push(assembleNode({ node, pos: pos.get(node.id), locales, viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id) }))
+    // focus screenshots of the keyboard walk (#19): only those of the current captures, not of earlier walks
+    const shots = new Set(caps.flatMap((c) => (c.cap.keyboard?.shots || []).map((s) => s.file)))
+    nodes.push(assembleNode({ node, pos: pos.get(node.id), locales, viewportNames, captures: caps, attachments: attachments.filter((a) => a.node === node.id && (a.kind !== 'focus' || shots.has(a.file))) }))
   }
   const summary = summarize(nodes, scenario.sessions)
 
@@ -126,19 +131,39 @@ async function buildStack ({ run, node, base, captures, attachments }) {
   return {
     ...base, status, by_locale, captures: {}, criteria, semantic: {}, metadata: {}, session: node.session || null,
     execution: node.execution ? { strategy: node.execution.strategy, instructions: node.execution.instructions } : { strategy: 'playwright', instructions: 'Open every page of the stack and capture it.' },
-    attachments: attachments.filter((a) => a.node === node.id).map((a) => ({ type: a.kind, title: a.title, file: a.file, locale: a.locale })),
+    attachments: attachments.filter((a) => a.node === node.id && a.kind !== 'focus').map((a) => ({ type: a.kind, title: a.title, file: a.file, locale: a.locale })),
     stack: { count: items.length, items: file, summary, fields, sample, semantic_sample: semanticSample, source: ['urls', 'file', 'sitemap', 'crawl', 'script'].find((k) => src[k]) || null, stale: items.filter((it) => it.stale).length }
   }
 }
 
+/**
+ * Lines for `kaloko build`: semantic criteria that failed or need a person, per step, locale and viewport, with the
+ * score and the one-sentence reason (#17). At most `max` lines.
+ */
+export function attentionLines (manifest, max = 20) {
+  const out = []
+  for (const n of manifest.nodes || []) {
+    for (const c of n.criteria || []) {
+      if (c.type !== 'semantic' || !['fail', 'needs_review'].includes(c.verdict)) continue
+      for (const [l, vps] of Object.entries(c.by_locale || {})) {
+        for (const [vp, r] of Object.entries(vps)) {
+          if (!['fail', 'needs_review'].includes(r.verdict)) continue
+          out.push(`  ${r.verdict === 'fail' ? '✕' : '?'} ${n.id} ${l}/${vp} ${c.id}${r.score != null ? ` ${Math.round(r.score * 100)} %${r.vision ? ' (with screenshot)' : ''}` : ''}${r.reason ? `: ${r.reason}` : r.note ? `: ${r.note}` : ''}`)
+        }
+      }
+    }
+  }
+  return out.length > max ? [...out.slice(0, max), `  … ${out.length - max} more on the canvas`] : out
+}
+
 export async function writePreview ({ run, manifest }) {
   const template = await readFile(join(packageRoot, 'viewer', 'index.html'), 'utf8')
   // the local preview opens from file://, where fetch cannot read files: stack indexes go inline
   // file:// cannot fetch: stack indexes and app UI trees travel inside the preview
   const withTrees = async (n) => {
-    if (!Object.values(n.captures || {}).some((vps) => Object.values(vps).some((c) => c?.tree))) return n
+    if (!Object.values(n.captures || {}).some((vps) => Object.values(vps).some((c) => c?.tree || c?.spec))) return n
     const captures = {}
-    for (const [l, vps] of Object.entries(n.captures)) { captures[l] = {}; for (const [vp, c] of Object.entries(vps)) captures[l][vp] = c?.tree ? { ...c, tree_inline: await readJson(join(run.dir, c.tree), null) } : c }
+    for (const [l, vps] of Object.entries(n.captures)) { captures[l] = {}; for (const [vp, c] of Object.entries(vps)) captures[l][vp] = c?.tree || c?.spec ? { ...c, ...(c.tree ? { tree_inline: await readJson(join(run.dir, c.tree), null) } : {}), ...(c.spec ? { spec_inline: await readJson(join(run.dir, c.spec), null) } : {}) } : c }
     return { ...n, captures }
   }
   // design runs: live steps open from the merged tree, where links between steps resolve (design.mjs copyLive)

package/src/lib/capture.mjs

@@ -2,6 +2,7 @@ import { writeFile, mkdir, rm } from 'node:fs/promises'
 import { dirname, relative, join } from 'node:path'
 import { buildHtmlView } from './htmlview.mjs'
 import { probe } from './probe.mjs'
+import { pageSpecScript } from './devspec.mjs'
 import { writeJson, readJsonl } from './util.mjs'
 import { addCapture, runPaths } from './run.mjs'
 
@@ -37,6 +38,15 @@ export const toWebp = (page, png, opts = {}) => encodeImage(page, png, { ...opts
  * Capture options: scenario-wide `capture:` merged with the node's. Documentation runs use
  * { format: png, scale: 2, mask: [selectors], annotate: true }; daily acceptance keeps the WebP default.
  */
+/** The page spec (src/lib/devspec.mjs), kept under ~250 kB so it fits a design revision's capture record. */
+export async function stepSpec (page, tokens = null) {
+  try {
+    const spec = await page.evaluate(pageSpecScript(tokens))
+    while (spec.elements.length > 20 && JSON.stringify(spec).length > 250000) spec.elements = spec.elements.slice(0, Math.floor(spec.elements.length * 0.8))
+    return spec
+  } catch { return null }
+}
+
 export function captureOptions (scenario, node) {
   const o = { ...(scenario?.capture || {}), ...(node?.capture || {}) }
   const q = Number(o.quality)
@@ -56,6 +66,8 @@ import { consoleEntriesFor } from './console.mjs'
 import { redact, redactDeep } from './secrets.mjs'
 import { readFields } from './stack.mjs'
 import { checkLinks, checkHreflang, requester, walkCache, NETWORK_ASSERTS, isolatedRequest, cacheScope } from './linkcheck.mjs'
+import { keyboardChecks, needsKeyboard, KEYBOARD_ASSERTS } from './keyboard.mjs'
+import { repeatRegions, resolveIgnore } from './stability.mjs'
 
 /** Probe arguments derived from the node's criteria. */
 export function probeArgs (node, maxChars, design = false) {
@@ -74,6 +86,9 @@ export function probeArgs (node, maxChars, design = false) {
 // md-menu, listboxes, tooltips). A step that leaves one open wants it on the capture.
 const OVERLAY = '[role=menu], [role=listbox], [role=tooltip], .MuiPopover-root, .MuiPopper-root, .MuiMenu-root, md-menu-content, .md-open-menu-container.md-active, .md-panel-outer-wrapper.md-panel-is-showing'
 
+const STACK_KEYBOARD = new Map()
+const STACK_KEYBOARD_PAGES = 5
+
 export async function settle (page) {
   try { await page.waitForLoadState('networkidle', { timeout: 5000 }) } catch {}
   try { await page.evaluate(() => document.fonts?.ready) } catch {}
@@ -95,7 +110,7 @@ export async function settle (page) {
  * Captures the current state of `page` for one step and locale in every
  * requested viewport: full-page PNG, rendered HTML, probe results.
  */
-export async function captureStep ({ page, node, locale, viewports, run, consoleFile, maxChars = 90000, since, extra = {}, format = null, options = null, designSummary = false, htmlView = null, item = null, fields = null }) {
+export async function captureStep ({ page, node, locale, viewports, run, consoleFile, maxChars = 90000, since, extra = {}, format = null, options = null, designSummary = false, htmlView = null, item = null, fields = null, tokens = null, stability = 0 }) {
   const opts = { ...(options || captureOptions(null, node)), ...(format ? { format } : {}) }
   // a page of a stack step lives under steps/<node>/items/<item>/…; its capture record names node and item
   const stepId = item ? `${node.id}/items/${item.id}` : node.id
@@ -132,6 +147,12 @@ export async function captureStep ({ page, node, locale, viewports, run, console
         throw error
       }
     }
+    // the step spec (Dev mode, `kaloko spec`, drift): elements with roles, names and styles mapped to the tokens
+    if (probeArgs(node, maxChars, designSummary).design && !item) result.spec = await stepSpec(page, tokens)
+    // regions the scenario ignores in pixel comparisons (#20), as page rectangles of this capture
+    const ignore = await resolveIgnore(page, node, locale, viewport)
+    // what moves while the page sits still (walk --stability N): repeated screenshots, compared here
+    const unstable = stability > 1 ? await repeatRegions(page, { n: Math.min(6, stability), fullPage: opts.fullPage }).catch((error) => ({ error: String(error.message).slice(0, 200) })) : null
     // numbered marks on the highlighted elements, in criteria order (first box of each)
     const annotations = opts.annotate ? Object.entries(result.boxes || {}).filter(([, list]) => list?.length).map(([criterion, list], i) => ({ n: i + 1, criterion, x: list[0].x, y: list[0].y })) : []
     let image = paths.step(stepId, locale, viewport, opts.format)
@@ -168,6 +189,24 @@ export async function captureStep ({ page, node, locale, viewports, run, console
           : { ok: errors.length === 0, evidence: errors.length ? errors.slice(0, 3).join(' | ') : 'no console errors' }
       }
     }
+    // keyboard and screen-reader checks (#19): a Tab walk and the accessibility tree, after the screenshot and after
+    // the console was read (focus handlers of the page must not count as its errors)
+    let keyboard = null
+    // stack steps walk the keyboard on their first few pages only (one template, hundreds of pages)
+    const kbKey = `${run.dir}|${node.id}|${viewport}`
+    const kbSampled = !item || (STACK_KEYBOARD.set(kbKey, (STACK_KEYBOARD.get(kbKey) || 0) + 1).get(kbKey) <= STACK_KEYBOARD_PAGES)
+    if (needsKeyboard(node) && !kbSampled) {
+      for (const c of node.criteria) if (KEYBOARD_ASSERTS.has(c.check.assert)) result.checks[c.id] = { ok: null, evidence: `keyboard walk runs on the first ${STACK_KEYBOARD_PAGES} pages of a stack step` }
+    }
+    if (needsKeyboard(node) && kbSampled) {
+      const kb = await keyboardChecks(page, node.criteria, {
+        saveShot: async (n, png) => { const f = paths.step(stepId, locale, viewport, `focus-${n}.png`); await writeFile(f, png); return relative(run.dir, f) }
+      })
+      Object.assign(result.checks, kb.checks)
+      result.boxes = { ...result.boxes, ...kb.boxes }
+      keyboard = kb.keyboard
+      for (const s of keyboard?.shots || []) await addCapture(run.dir, { type: 'attachment', kind: 'focus', node: node.id, ...(item ? { item: item.id } : {}), locale, viewport, title: `Focus ${s.n}: ${s.name}`, file: s.file, at: new Date().toISOString() })
+    }
     // experimental HTML view (#42): a scrubbed, script-free static copy with rehosted assets
     let view = null
     if (htmlView) {
@@ -189,6 +228,9 @@ export async function captureStep ({ page, node, locale, viewports, run, console
       ...extra,
       http_status: extra.http_status ?? result.http_status ?? null,
       capture: { format: String(image).split('.').pop(), scale: opts.scale, masked: opts.mask.length, annotations, ...(opts.fullPage ? {} : { full_page: false }) },
+      ...(ignore.length ? { ignore } : {}),
+      ...(unstable ? { unstable } : {}),
+      ...(keyboard ? { keyboard } : {}),
       ...(item ? { item: { id: item.id, path: item.path, ...(item.title ? { title: item.title } : {}), ...(item.purpose ? { purpose: item.purpose } : {}) } } : {}),
       ...(fields ? { fields: { ...(await page.evaluate(readFields, fields).catch(() => ({}))), ...(fields.some((f) => (f.from || f.id) === 'http') ? Object.fromEntries(fields.filter((f) => (f.from || f.id) === 'http').map((f) => [f.id, extra.http_status ?? result.http_status ?? null])) : {}) } } : {}),
       files: { png: relative(run.dir, image), image: relative(run.dir, image), thumb: thumbWritten ? relative(run.dir, thumb) : null, html: relative(run.dir, html) }

package/src/lib/design-cli.mjs

@@ -7,6 +7,7 @@ import { readJson, writeJson, UserError } from './util.mjs'
 import { readCaptures, addCapture, runPaths } from './run.mjs'
 import { latestCaptures, criteriaHash, evaluationSource } from './evaluate.mjs'
 import { captureStep } from './capture.mjs'
+import { loadTokens } from './drift.mjs'
 import { layout } from './scenario.mjs'
 import { launchEphemeral } from './browser.mjs'
 import { observeConsole } from './console.mjs'
@@ -45,6 +46,8 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
   const browser = await launchEphemeral(config)
   const paths = runPaths(run.dir)
   const mode = env.tokens_mode || ''
+  // the design folder's own set names the values in the step spec (the one its live pages reference)
+  const tokens = await loadTokens({ client: null, config, root, designDir: dir })
   let rendered = 0, reused = 0
   try {
     const page = await (await browser.newContext()).newPage()
@@ -58,7 +61,8 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
       for (const [vp, size] of Object.entries(viewports)) {
         const ck = captureKey(rev.id, vp, mode)
         const cached = noCache ? null : await fromCache({ root, output: config.output, key: ck })
-        if (cached && cached.record.criteria_hash === criteriaHash(node)) {
+        // a capture from before the step spec existed renders again, so Dev mode and `kaloko spec` have it
+        if (cached && cached.record.criteria_hash === criteriaHash(node) && cached.record.spec) {
           const record = await restoreCached({ cached, runDir: run.dir, paths, node: node.id, locale: rev.locale, viewport: vp })
           const file = paths.step(node.id, rev.locale, vp, 'capture.json')
           await writeJson(file, record)
@@ -69,7 +73,7 @@ export async function walkDesign ({ config, root, run, scenario, env, pick, loca
       if (Object.keys(todo).length) {
         const since = new Date().toISOString()
         await page.goto(`${server.url}/${rev.entry}`, { waitUntil: 'load' })
-        const results = await captureStep({ page, node, locale: rev.locale, viewports: todo, run, maxChars: config.evaluators?.jev?.max_state_chars, since, extra: { live }, designSummary: true })
+        const results = await captureStep({ page, node, locale: rev.locale, viewports: todo, run, maxChars: config.evaluators?.jev?.max_state_chars, since, extra: { live }, designSummary: true, tokens })
         for (const r of results) { await toCache({ root, output: config.output, key: captureKey(rev.id, r.viewport, mode), runDir: run.dir, record: r }); rendered++ }
       }
       say(`✓ ${node.id} ${rev.locale}  revision ${rev.id.slice(0, 8)}${Object.keys(todo).length ? '' : '  (unchanged, reused)'}`)
@@ -231,7 +235,7 @@ export async function syncDesign ({ config, root, run, scenario, env, client, ar
           out[k] = h
         }
         const stored = await readJson(runPaths(run.dir).step(rev.node, rev.locale, vp, 'eval.json'), null)
-        out.record = slimCapture(rec)
+        out.record = { ...slimCapture(rec), ...(rec.spec ? { spec: rec.spec } : {}) }
         out.evaluation = stored?.source === evaluationSource(node, rec) ? stored : null
         out.mode = env.tokens_mode || ''
         out.renderer = RENDERER