kaloko 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +92 -5
  2. package/package.json +1 -1
  3. package/schemas/config.v1.json +33 -0
  4. package/schemas/scenario.v1.json +39 -0
  5. package/schemas/spec.v1.json +60 -0
  6. package/skills/kaloko/SKILL.md +24 -3
  7. package/skills/kaloko/reference/acceptance.md +50 -2
  8. package/skills/kaloko/reference/ci.md +70 -0
  9. package/skills/kaloko/reference/collaboration.md +9 -1
  10. package/skills/kaloko/reference/design.md +35 -2
  11. package/skills/kaloko/reference/environments.md +5 -0
  12. package/skills/kaloko/reference/evaluation.md +46 -3
  13. package/skills/kaloko/reference/execution.md +74 -0
  14. package/skills/kaloko/reference/import.md +66 -0
  15. package/skills/kaloko/reference/mcp.md +4 -1
  16. package/src/cli.mjs +428 -99
  17. package/src/lib/affected-core.mjs +242 -0
  18. package/src/lib/affected.mjs +233 -0
  19. package/src/lib/build.mjs +30 -5
  20. package/src/lib/capture.mjs +43 -1
  21. package/src/lib/ci.mjs +207 -0
  22. package/src/lib/config.mjs +2 -0
  23. package/src/lib/design-cli.mjs +7 -3
  24. package/src/lib/devspec.mjs +821 -0
  25. package/src/lib/doctor.mjs +7 -0
  26. package/src/lib/drift.mjs +16 -8
  27. package/src/lib/evaluate.mjs +98 -14
  28. package/src/lib/git.mjs +6 -2
  29. package/src/lib/heal.mjs +207 -0
  30. package/src/lib/import.mjs +374 -0
  31. package/src/lib/init-detect.mjs +219 -0
  32. package/src/lib/keyboard.mjs +261 -0
  33. package/src/lib/manifest-core.mjs +16 -5
  34. package/src/lib/onboarding.mjs +15 -8
  35. package/src/lib/pack-help.mjs +7 -0
  36. package/src/lib/packs.mjs +9 -2
  37. package/src/lib/png.mjs +61 -0
  38. package/src/lib/pr.mjs +24 -2
  39. package/src/lib/probe.mjs +57 -0
  40. package/src/lib/recheck.mjs +3 -1
  41. package/src/lib/scenario.mjs +2 -1
  42. package/src/lib/service.mjs +7 -1
  43. package/src/lib/spec-cli.mjs +90 -0
  44. package/src/lib/stability-core.mjs +114 -0
  45. package/src/lib/stability.mjs +249 -0
  46. package/src/lib/unzip.mjs +31 -0
  47. package/src/lib/upload.mjs +114 -0
  48. package/src/lib/vision-core.mjs +132 -0
  49. package/src/lib/vision.mjs +127 -0
  50. package/templates/kaloko.config.yml +6 -0
  51. package/viewer/index.html +500 -44
package/src/cli.mjs CHANGED
@@ -1,9 +1,9 @@
1
- import { writeFile, access, readFile, mkdir } from 'node:fs/promises'
1
+ import { writeFile, access, readFile, mkdir, rm } from 'node:fs/promises'
2
2
  import { existsSync } from 'node:fs'
3
- import { gzipSync } from 'node:zlib'
4
- import { createHash } from 'node:crypto'
5
3
  import { join, resolve, relative, dirname } from 'node:path'
6
- import { pathToFileURL } from 'node:url'
4
+ import { pathToFileURL, fileURLToPath } from 'node:url'
5
+ import { AsyncLocalStorage } from 'node:async_hooks'
6
+ import { availableParallelism } from 'node:os'
7
7
  import yaml from 'js-yaml'
8
8
  import { loadConfig, environment, urlFor, baseUrl, CONFIG_FILE, LEGACY_CONFIG_FILE } from './lib/config.mjs'
9
9
  import { loadScenario, validateScenario } from './lib/scenario.mjs'
@@ -17,7 +17,8 @@ import { newContext, accountFor, gateHeaders, gateOrigins, originMatcher, gateFo
17
17
  import { configureGate } from './lib/linkcheck.mjs'
18
18
  import { rememberEnvironment } from './lib/secrets.mjs'
19
19
  import { evaluateRun, semanticPrefetch } from './lib/evaluate.mjs'
20
- import { buildManifest, writePreview } from './lib/build.mjs'
20
+ import { buildManifest, writePreview, attentionLines } from './lib/build.mjs'
21
+ import { stabilityCmd } from './lib/stability.mjs'
21
22
  import { localized, readJson, UserError, globToRegExp, failureStreak, expiryNote } from './lib/util.mjs'
22
23
  import { packageRoot } from './lib/schema.mjs'
23
24
  import { compareRuns, formatComparison } from './lib/compare.mjs'
@@ -25,6 +26,7 @@ import { doctor, formatDoctor } from './lib/doctor.mjs'
25
26
  import { parsePlan, coverage, formatCoverage } from './lib/coverage.mjs'
26
27
  import { calibrate, formatCalibration, casesFromGolden, casesFromRun } from './lib/calibrate.mjs'
27
28
  import { pageFacts, draftScenario, scenarioYaml } from './lib/draft.mjs'
29
+ import { detectProject, initScenario, formatDetection } from './lib/init-detect.mjs'
28
30
  import { recheckCaptures } from './lib/recheck.mjs'
29
31
  import { resolveItems, fieldsOf, DEFAULT_LIMIT } from './lib/stack.mjs'
30
32
  import { DEFAULTS as JEV_DEFAULTS } from './lib/jev.mjs'
@@ -35,6 +37,13 @@ import { htmlViewAllowed } from './lib/htmlview.mjs'
35
37
  import { service, serviceUrl } from './lib/service.mjs'
36
38
  import { login } from './lib/login.mjs'
37
39
  import { initNextSteps, loginNextSteps } from './lib/onboarding.mjs'
40
+ import { specCmd, driftCmd } from './lib/spec-cli.mjs'
41
+ import { hashFiles, missingHashes, uploadRunFiles, PROGRESS_FILE, fmtBytes } from './lib/upload.mjs'
42
+ import { affectedFor, carryCaptures } from './lib/affected.mjs'
43
+ import { formatAffected, selectedIds } from './lib/affected-core.mjs'
44
+ import { healStep, formatHeal } from './lib/heal.mjs'
45
+ import { readSource, stepFor, viewportFor, adhocScenario, writeImported, attachImported, keepVideos, slug } from './lib/import.mjs'
46
+ import { findPreview, pullRequestOf, envLines, writeEnvFile, githubWorkflow, gitlabJob, addPreviewEnvironment } from './lib/ci.mjs'
38
47
  import { walkDesign, syncDesign, publishVersion, historyCmd, referenceCmd, tokensCmd, exportLive, localRevisions, branchCmd, uploadBlobs } from './lib/design-cli.mjs'
39
48
  import { designDir, readBase, headKey } from './lib/design.mjs'
40
49
  import { lintDesign, formatLint } from './lib/design-lint.mjs'
@@ -43,7 +52,6 @@ import { driverFor, isApp, pngSize, ocr as ocrImage } from './lib/apps/drivers.m
43
52
  import { captureApp } from './lib/apps/capture.mjs'
44
53
  import { readTree, flatten } from './lib/apps/tree.mjs'
45
54
 
46
- const fmtBytes = (n) => { n = Number(n) || 0; const u = ['B', 'kB', 'MB', 'GB']; let i = 0; while (n >= 1024 && i < u.length - 1) { n /= 1024; i++ } return `${n.toFixed(i ? 1 : 0)} ${u[i]}` }
47
55
 
48
56
  const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
49
57
 
@@ -53,8 +61,18 @@ Project
53
61
  draft <url> [url…] [--id SITE-X] [--title t] [--out qa/flows/site.yml] [--force]
54
62
  First plan + read-only production scenario for public pages, drafted from the
55
63
  pages (title, description, H1) with the seo, a11y, perf and console packs
56
- init [--agent claude|codex|all] [--org <slug>]
57
- Create ${CONFIG_FILE} and install the kaloko skill for your agent
64
+ init [--agent claude|codex|all] [--org <slug>] [--no-detect] [--no-scenario]
65
+ Create ${CONFIG_FILE} and install the kaloko skill for your agent; detects the
66
+ framework, dev server URL, locales and main routes, and drafts a first scenario
67
+ (qa/flows/main-pages.yml) when the project has none
68
+ ci github|gitlab [--scenario a.yml,b.yml] [--env preview] [--trigger deployment|pull_request] [--no-affected]
69
+ [--out file] [--print] [--force]
70
+ Write a workflow that walks what each pull request affects on its preview
71
+ deployment (Vercel, Netlify, Cloudflare Pages, review apps) and comments on it;
72
+ adds the preview environment (base_url \${KALOKO_PREVIEW_URL}) to the config
73
+ ci preview-url [--wait 600] [--github-env] [--dotenv file]
74
+ In CI: the preview URL (KALOKO_PREVIEW_URL, the deployment event, provider
75
+ variables or the commit's GitHub deployments), the pull request and its base
58
76
  login [--org <slug>] [--url <service>] [--print] [--no-open]
59
77
  Sign this machine in through the browser: approve the code on kaloko.app, choose
60
78
  the organization and what the token may do; the token (at most 30 days) goes to
@@ -72,6 +90,7 @@ Run
72
90
  Create a run directory and make it current
73
91
  browser start|stop|status [--env e] [--headed] One shared browser per environment (port = browser.port + index)
74
92
  Shared Chromium with CDP (agents, scripts and people drive it)
93
+ browser install [--with-deps] Download the Chromium of kaloko's Playwright (CI; KALOKO_BROWSER_CHANNEL=chromium uses it)
75
94
  open --step <id> [--locale l] Navigate the current page to the step URL
76
95
  session reset New visitor: clear cookies and storage in the shared browser
77
96
  capture --step <id> [--locale l] [--url <url>] [--page <substring>] [--viewports a,b]
@@ -101,6 +120,24 @@ Run
101
120
  verdicts and what failed (--no-evaluate skips that; kaloko evaluate redoes it)
102
121
  [--items a,b] [--limit N] Stack steps (kind: stack, many pages of one template): only pages whose id or
103
122
  path contains a, b; at most N pages (default items.limit, 500)
123
+ [--stability N] Take N screenshots of each capture a moment apart and note regions that move
124
+ (clocks, carousels); kaloko stability turns them into ignore suggestions
125
+ [--concurrency N] Locales at the same time, each in its own browser context (default: by CPU in an
126
+ own browser, 1 on the shared browser; walk.concurrency in the config)
127
+ [--affected [--base ref] [--no-upstream] [--if-unmapped all] [--carry]]
128
+ Only the steps the changed files reach (see kaloko affected), plus steps on the
129
+ main path before them and steps with affects: always; --carry copies the other
130
+ steps' captures from the previous local run
131
+ [--heal | --no-heal] A selector the page lost gets a proposal from the page (test id, role and name,
132
+ text) with a patch; --heal applies it to the step script once the step passes
133
+ affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--check] [--offline]
134
+ Which steps changed files reach, and why: framework route files against step paths
135
+ and URLs captured earlier, imports, global files, i18n messages, step scripts and
136
+ affects: globs; --check exits 1 when nothing is affected
137
+ import <folder | playwright-report/ | results.json | trace.zip | walkthrough.md> [--scenario file | --adhoc]
138
+ [--env e] [--locale l] [--viewport v] [--map "Title=step,…"] [--format f] [--out file]
139
+ Screenshots, recordings and traces from agents and tools as a run: into the
140
+ current run, a new run of --scenario, or an ad-hoc flow of the screens
104
141
  status Show what the current run has captured
105
142
  Scenarios may declare "sessions" (people in the flow); walk keeps one browser context per session and locale
106
143
 
@@ -111,8 +148,10 @@ Results
111
148
  console, overflow and performance keep their live result
112
149
  build Write run.json and the local preview (index.html)
113
150
  preview Build and print the preview path
114
- share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role);
115
- --pr adds or updates one comment on the branch's pull request via gh
151
+ share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
152
+ hash, so unchanged captures are not sent again and a broken-off share resumes;
153
+ --pr adds or updates one comment on the branch's pull request via gh (KALOKO_PR in
154
+ CI), or a note on the GitLab merge request (KALOKO_GITLAB_TOKEN)
116
155
  feedback [--run id] [--scenario id] [--json]
117
156
  Open threads from reviewers (steps, the whole run, the whole flow) with ids and replies;
118
157
  returned steps first, with their notes
@@ -143,6 +182,10 @@ Results
143
182
  Newest capture of every step as steps/<step>/<locale>/<viewport>.<ext> plus
144
183
  index.json (documentation layer, Business plan); unchanged files are skipped
145
184
  public [--run id] [--off] Make a shared run readable without sign-in (admin; testers may refresh an already public scenario)
185
+ stability [--last 4 | --runs a,b] [--same-commit] [--apply] [--json]
186
+ Regions that change on every run or move while the page is idle (clocks, ads,
187
+ carousels): ignore suggestions per step in <run>/stability.yml; --apply adds
188
+ them to the scenario (ignore:), and pixel diffs then skip them
146
189
  compare [--run id] [--baseline id | --design] [--json]
147
190
  Text diff of two runs: step statuses, criterion verdicts, scores and evidence
148
191
  (default: current run against the scenario baseline, else the previous run)
@@ -160,6 +203,12 @@ Design (environments with kind: design; live HTML steps in a folder)
160
203
  share [--name <text>] [--branch b] Publish the draft as a version of the flow (a run): "v4 · 3 changed, 17 unchanged"
161
204
  history [--scenario id] [--step id] [--locale l] [--json]
162
205
  Versions of the flow; revisions of a step with who (person or agent) and when
206
+ spec [step] [--ref design|draft|vN|<run id>] [--locale l] [--viewport v] [--local] [--json]
207
+ What to build for a step as approved: elements with role, name and selector, styles
208
+ mapped to DTCG tokens (or the nearest one), :hover/:focus/:active/:disabled, contrast
209
+ drift [step] [--ref design|vN] [--remote <run id|latest>] [--json]
210
+ How this implementation run differs from the approved design (expected token vs actual
211
+ value, raw values, missing elements and states); exit code 1 when something differs
163
212
  reference [--scenario id] (--design|--baseline) (--run id|--version n|--off) [--branch b]
164
213
  Show or set the design reference (accepted version) and the baseline
165
214
  tokens pull|push|check|validate [--project p] [--file tokens.json|tokens.resolver.json] [--mode m] [--json]
@@ -223,7 +272,40 @@ function parse (argv) {
223
272
  return args
224
273
  }
225
274
  const list = (v) => (typeof v === 'string' ? v.split(',').map((s) => s.trim()).filter(Boolean) : null)
226
- const say = (...m) => console.log(...m)
275
+ // parallel walks (#27) print each locale's lines together, in locale order: `say` writes into the running unit's buffer
276
+ const output = new AsyncLocalStorage()
277
+ const say = (...m) => { const buf = output.getStore(); if (buf) buf.push(m.join(' ')); else console.log(...m) }
278
+
279
+ /**
280
+ * Runs `job` over `items`, `n` at a time, and returns the results in item order. The first item prints live; the
281
+ * others print in item order once every earlier one is done, so the log reads the same however the work interleaved.
282
+ */
283
+ export async function inOrder (items, n, job, print = (l) => console.log(l)) {
284
+ const results = new Array(items.length)
285
+ if (n <= 1) { for (const [i, it] of items.entries()) results[i] = await job(it); return results }
286
+ const bufs = items.map(() => []); const finished = items.map(() => false)
287
+ let printed = 1
288
+ const flush = () => { while (printed < items.length && finished[printed - 1] && finished[printed]) { for (const l of bufs[printed]) print(l); printed++ } }
289
+ const queue = items.map((_, i) => i)
290
+ const worker = async () => {
291
+ for (let i = queue.shift(); i !== undefined; i = queue.shift()) {
292
+ results[i] = await (i === 0 ? job(items[i]) : output.run(bufs[i], () => job(items[i])))
293
+ finished[i] = true
294
+ flush()
295
+ }
296
+ }
297
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, worker))
298
+ flush()
299
+ return results
300
+ }
301
+
302
+ /** Locales walked at once: --concurrency, walk.concurrency, else by CPU (one at a time on the shared browser). */
303
+ export function walkConcurrency (args, config, units, shared, cpus = availableParallelism()) {
304
+ const asked = Number(args.concurrency) || Number(config.walk?.concurrency)
305
+ if (asked) return Math.max(1, Math.min(16, Math.floor(asked)))
306
+ if (units < 2 || shared) return 1
307
+ return Math.max(1, Math.min(units, 4, Math.floor(cpus / 2)))
308
+ }
227
309
 
228
310
  async function context (args) {
229
311
  const { config, root, file } = await loadConfig(process.cwd(), args.config)
@@ -522,6 +604,13 @@ async function captureImage ({ config, run, scenario, node, locale, args }) {
522
604
  } finally { await enc?.close().catch(() => {}) }
523
605
  }
524
606
 
607
+ /** walk --affected / kaloko affected: the selection for one scenario (a token adds URLs of the newest shared run). */
608
+ async function affectedSelection (args, { config, root, scenario, scenarioFile }) {
609
+ let client = null
610
+ try { if (!args.offline) client = service(config) } catch {}
611
+ return affectedFor({ root, config, scenario, scenarioFile, base: typeof args.base === 'string' ? args.base : null, files: list(args.files), client, upstream: !args['no-upstream'] })
612
+ }
613
+
525
614
  const commands = {
526
615
  async init (args) {
527
616
  const cwd = process.cwd()
@@ -530,14 +619,22 @@ const commands = {
530
619
  try { await access(file); exists = true } catch {}
531
620
  // a project from before the rename keeps its qawalk.config.yml (it is still read)
532
621
  try { await access(join(cwd, LEGACY_CONFIG_FILE)); exists = true; say(`${LEGACY_CONFIG_FILE} found; kept (still read; rename it to ${CONFIG_FILE} when convenient).`) } catch {}
622
+ // what the project is (#26): framework, dev server, locales, main routes; --no-detect keeps the plain template
623
+ const detected = args['no-detect'] ? null : await detectProject(cwd)
624
+ if (detected) say(formatDetection(detected))
533
625
  if (!exists) {
534
626
  let tpl = await readFile(join(packageRoot, 'templates', 'kaloko.config.yml'), 'utf8')
535
627
  if (typeof args.org === 'string') {
536
628
  if (!/^[a-z0-9][a-z0-9-]{1,40}$/.test(args.org)) throw new UserError('--org must be the organization slug from kaloko.app (a-z, 0-9, -).')
537
629
  tpl = tpl.replace(/^org: .*$/m, `org: ${args.org}`)
538
630
  }
631
+ if (detected) {
632
+ const project = (detected.name || cwd.split('/').pop() || '').toLowerCase().replace(/^@[^/]+\//, '').replace(/[^a-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '')
633
+ if (project) tpl = tpl.replace(/^project: .*$/m, `project: ${project}`)
634
+ tpl = tpl.replace(' local:\n base_url:\n default: http://localhost:3000', ` local:\n # dev server: ${detected.command || 'start your app'}${detected.framework ? ` (${detected.framework.name}, found by kaloko init)` : ''}\n base_url:\n default: ${detected.url}`)
635
+ }
539
636
  await writeFile(file, tpl)
540
- say(`Created ${CONFIG_FILE}. Edit org and environments.`)
637
+ say(`Created ${CONFIG_FILE}${detected ? ` (local: ${detected.url})` : ''}. Edit org and the staging and production environments.`)
541
638
  } else if (!existsSync(join(cwd, LEGACY_CONFIG_FILE))) say(`${CONFIG_FILE} already exists; kept.`)
542
639
  const agent = typeof args.agent === 'string' ? args.agent : 'none'
543
640
  if (!['claude', 'codex', 'all', 'none'].includes(agent)) throw new UserError('--agent must be claude, codex, all or none.')
@@ -570,12 +667,32 @@ const commands = {
570
667
  await cp(join(packageRoot, 'templates', 'steps', 'consent.mjs'), consentStep)
571
668
  say('Added qa/flows/steps/consent.mjs (accepts the consent dialog, if the site has one)')
572
669
  }
670
+ // a first scenario that walks as it is, when the project has none yet
671
+ let first = null
672
+ if (detected && !args['no-scenario']) {
673
+ try {
674
+ const { config, root } = await loadConfig(cwd)
675
+ if (!(await scenarioFiles(config, root)).length) {
676
+ const scenario = initScenario(detected)
677
+ const problems = validateScenario(JSON.parse(JSON.stringify(scenario)))
678
+ if (problems.length) throw new Error(problems.join('; '))
679
+ first = join(cwd, 'qa', 'flows', 'main-pages.yml')
680
+ await mkdir(join(cwd, 'qa', 'flows', 'steps'), { recursive: true })
681
+ await writeFile(first, `# First scenario drafted by kaloko init from ${detected.routes.length ? 'the route files' : 'the home page'}; sharpen purposes and criteria, add the flows that matter.\n${scenarioYaml(scenario)}`)
682
+ const open = join(cwd, 'qa', 'flows', 'steps', 'open.mjs')
683
+ if (!existsSync(open)) await writeFile(open, "// Opens the step's page (kaloko init).\nexport default async ({ page, url }) => { await page.goto(url(), { waitUntil: 'domcontentloaded' }) }\n")
684
+ say(`Drafted ${relative(cwd, first)}: ${scenario.nodes.length} page(s) in ${scenario.locales.join(', ')}`)
685
+ }
686
+ } catch (e) { say(`No scenario drafted: ${e.message.split('\n')[0]}`) }
687
+ }
688
+ let running = false
689
+ if (detected) { try { const r = await fetch(detected.url, { signal: AbortSignal.timeout(1500) }); running = r.status < 500 } catch {} }
573
690
  // the organization's sample run and the way to a first real flow (src/lib/onboarding.mjs)
574
691
  let written = ''
575
692
  try { written = await readFile(existsSync(file) ? file : join(cwd, LEGACY_CONFIG_FILE), 'utf8') } catch {}
576
693
  const org = typeof args.org === 'string' ? args.org : (written.match(/^org:\s*["']?([a-z0-9-]+)["']?\s*$/m) || [])[1]
577
694
  const base = serviceUrl({ share: { url: (written.match(/^share:[ \t]*\n(?:[ \t]+.*\n)*?[ \t]+url:[ \t]*["']?(https?:\/\/[^\s"']+)/m) || [])[1] } })
578
- say(initNextSteps({ base, org }).join('\n'))
695
+ say(initNextSteps({ base, org, loggedIn: !!process.env.KALOKO_TOKEN, devServer: detected && !running ? detected : null, scenario: first && relative(cwd, first) }).join('\n'))
579
696
  say(`Scenario template: ${join(packageRoot, 'templates', 'scenario.yml')}`)
580
697
  },
581
698
 
@@ -633,6 +750,14 @@ const commands = {
633
750
  async browser (args) {
634
751
  const { config, root } = await context(args)
635
752
  const action = args._[0] || 'status'
753
+ if (action === 'install') {
754
+ // the Chromium of the Playwright version kaloko itself uses (CI: --with-deps adds the system libraries)
755
+ const { spawn } = await import('node:child_process')
756
+ const cli = join(dirname(fileURLToPath(import.meta.resolve('playwright/package.json'))), 'cli.js')
757
+ const code = await new Promise((done) => spawn(process.execPath, [cli, 'install', ...(args['with-deps'] ? ['--with-deps'] : []), 'chromium'], { stdio: 'inherit' }).on('close', done))
758
+ if (code) throw new UserError(`Installing Chromium failed (exit ${code}).`)
759
+ return
760
+ }
636
761
  let env = null
637
762
  if (typeof args.env === 'string') env = environment(config, args.env)
638
763
  else { try { const run = await resolveRun({ root, output: config.output, run: args.run }); env = environment(config, run.meta.environment.name) } catch {} }
@@ -704,7 +829,7 @@ const commands = {
704
829
  return
705
830
  }
706
831
  const since = (await readCaptures(run.dir)).filter((c) => c.node === node.id).map((c) => c.at).pop()
707
- const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null })
832
+ const results = await captureStep({ page, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, list(args.viewports)), run, consoleFile: conn ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, since, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, stability: Number(args.stability) || 0 })
708
833
  for (const r of results) {
709
834
  const failed = Object.entries(r.checks).filter(([, v]) => v.ok === false).map(([k]) => k)
710
835
  say(`✓ ${node.id} ${locale} ${r.viewport} ${r.size.width}×${r.size.height}px ${failed.length ? `failing checks: ${failed.join(', ')}` : 'checks ok'}${r.blocked_requests.length ? ` blocked: ${r.blocked_requests.length}` : ''}`)
@@ -738,6 +863,24 @@ const commands = {
738
863
 
739
864
  async walk (args) {
740
865
  const { config, root, run, scenario, env } = await runContext(args)
866
+ if (args.affected) {
867
+ // only the steps the changes reach (#14), explained; the rest stays as it was (or is carried with --carry)
868
+ const r = await affectedSelection(args, { config, root, scenario, scenarioFile: resolve(root, run.meta.scenario_file) })
869
+ say(formatAffected(r.sel, { scenario, base: r.base, changed: r.files.length }))
870
+ if (args['if-unmapped'] === 'all' && r.sel.unmapped.length) say('Unmapped changes: walking every step (--if-unmapped all).\n')
871
+ else {
872
+ let ids = selectedIds(scenario, r.sel)
873
+ if (list(args.steps)) ids = ids.filter((id) => list(args.steps).includes(id))
874
+ if (!ids.length || r.sel.none) { say('Nothing to walk: no change reaches a step of this scenario.'); return }
875
+ args.steps = ids.join(',')
876
+ if (r.sel.locales && !args.locales) { const ls = r.sel.locales.filter((l) => run.meta.scope.locales.includes(l)); if (ls.length) args.locales = ls.join(',') }
877
+ if (args.carry) {
878
+ const c = await carryCaptures({ root, output: config.output, run, scenario, walked: new Set(ids) })
879
+ say(c ? `Carried ${c.steps.length} step(s) not walked from run ${c.from}: ${c.steps.join(', ')}` : 'Nothing to carry: no earlier local run of this scenario on this environment.')
880
+ }
881
+ say('')
882
+ }
883
+ }
741
884
  if (env.kind === 'design') {
742
885
  await walkDesign({ config, root, run, scenario, env, pick: list(args.steps), locales: list(args.locales) || run.meta.scope.locales, viewports: scopeViewports(run, list(args.viewports)), say, noCache: !!args['no-cache'] })
743
886
  const lint = await lintDesign({ dir: designDir(env, root), scenario })
@@ -752,17 +895,20 @@ const commands = {
752
895
  const conn = args.ephemeral ? null : await connectBrowser({ root, config, env })
753
896
  const releaseLock = conn ? await acquireWalkLock({ root, config, env, run: run.meta.id }).catch(async (e) => { await conn.browser.close(); throw e }) : null
754
897
  const ephemeral = conn ? null : await launchEphemeral(config)
755
- const ctx = conn ? conn.context : await newContext(ephemeral, env, { root, config })
898
+ const browser = conn ? conn.browser : ephemeral
899
+ // locales are independent: with --concurrency > 1 each walks in its own browser context at the same time (#27);
900
+ // one locale at a time keeps the shared browser's own context, as before
901
+ const concurrency = walkConcurrency(args, config, locales.length, !!conn)
902
+ const parallel = concurrency > 1 && locales.length > 1
903
+ const ctx = parallel ? null : conn ? conn.context : await newContext(ephemeral, env, { root, config })
756
904
  const scenarioDir = resolve(root, run.meta.scenario_file, '..')
757
905
  // semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
758
906
  const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root })
759
- const pending = []
760
- const failed = []
761
- const skipped = []
762
907
  const walked = await readCaptures(run.dir)
763
908
  const sessions = scenario.sessions || null
764
909
  // named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
765
910
  const sessionContexts = new Map()
911
+ const ownContexts = []
766
912
  // `account` stays the scenario's key; `credentials` resolves it in this environment (config accounts)
767
913
  const sessionInfo = (key) => key && sessions?.[key] ? { key, title: localized(sessions[key].title, locales[0]), mailbox: sessions[key].mailbox ? expandRun(sessions[key].mailbox, run.meta.short) : null, account: sessions[key].account || null, credentials: accountFor(env, sessions[key].account, { root, config }) } : null
768
914
  const contextFor = async (key, locale) => {
@@ -771,7 +917,7 @@ const commands = {
771
917
  if (!sessionContexts.has(id)) {
772
918
  // every context gets the environment's gate and readonly policy; an account's saved sign-in is loaded
773
919
  const account = accountFor(env, sessions[key].account, { root, config })
774
- const c = await newContext(conn ? conn.browser : ephemeral, env, { root, config, account })
920
+ const c = await newContext(browser, env, { root, config, account })
775
921
  const p = await c.newPage()
776
922
  observeConsole(p)
777
923
  sessionContexts.set(id, { context: c, page: p })
@@ -783,76 +929,249 @@ const commands = {
783
929
  // stop a session after the same failure repeats (shared precondition such as a failed sign-in);
784
930
  // --keep-going walks every step regardless
785
931
  const streak = failureStreak(args['keep-going'] ? 0 : 3)
786
- try {
787
- for (const locale of locales) {
788
- // the walk owns its tab; agents and people keep theirs in the same shared browser
789
- const page = await ctx.newPage()
790
- if (!conn) observeConsole(page)
791
- try {
792
- // every locale starts as a new visitor on its first walk in the run
793
- if (conn && !walked.some((c) => c.locale === locale)) { await resetSession(ctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale})`) }
794
- for (const node of scenario.nodes) {
795
- if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
796
- if (node.kind !== 'stack' && node.path && !urlFor(env, node, locale)) continue // step does not exist in this locale
797
- if (node.session && sessions?.[node.session]?.blocked_on?.includes(env.name)) { pending.push(`${node.id} (${locale}): session ${node.session} is blocked on ${env.name}`); continue }
798
- const sess = await contextFor(node.session, locale)
799
- const sPage = sess ? sess.page : page
800
- const sCtx = sess ? sess.context : ctx
801
- if (node.kind === 'stack') {
802
- try {
803
- const bad = await walkStack({ node, locale, page: sPage, env, run, scenario, scenarioDir, config, consoleFile: conn && !sess ? consoleLog(root, config.output, env.name) : null, args })
804
- for (const b of bad) failed.push(`${node.id} (${locale}) ${b}`)
805
- } catch (error) {
806
- failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
807
- say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
808
- }
809
- continue
810
- }
811
- if (node.kind === 'email' && (node.mail?.to || sessionInfo(node.session)?.mailbox)) {
812
- // an e-mail node with a recipient is walked like a step: wait for the message and capture it
813
- try {
814
- const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
815
- const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
816
- say(`✓ ${node.id} ${locale}: "${results.subject}"`)
817
- } catch (error) {
818
- failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
819
- say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
820
- }
821
- continue
932
+ const proposals = []
933
+ // one locale: its own tab (and, in parallel, its own context); returns what it could not walk
934
+ const walkLocale = async (locale) => {
935
+ const pending = []; const failed = []; const skipped = []
936
+ let lctx = ctx
937
+ if (parallel) { lctx = await newContext(browser, env, { root, config }); ownContexts.push(lctx) }
938
+ // the walk owns its tab; agents and people keep theirs in the same shared browser
939
+ const page = await lctx.newPage()
940
+ const shared = conn && !parallel // the shared browser's own context logs its console to a file
941
+ if (!shared) observeConsole(page)
942
+ try {
943
+ // every locale starts as a new visitor on its first walk in the run (an own context is new already)
944
+ if (shared && !walked.some((c) => c.locale === locale)) { await resetSession(lctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale})`) }
945
+ for (const node of scenario.nodes) {
946
+ if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
947
+ if (node.kind !== 'stack' && node.path && !urlFor(env, node, locale)) continue // step does not exist in this locale
948
+ if (node.session && sessions?.[node.session]?.blocked_on?.includes(env.name)) { pending.push(`${node.id} (${locale}): session ${node.session} is blocked on ${env.name}`); continue }
949
+ const sess = await contextFor(node.session, locale)
950
+ const sPage = sess ? sess.page : page
951
+ const sCtx = sess ? sess.context : lctx
952
+ const consoleFile = shared && !sess ? consoleLog(root, config.output, env.name) : null
953
+ if (node.kind === 'stack') {
954
+ try {
955
+ const bad = await walkStack({ node, locale, page: sPage, env, run, scenario, scenarioDir, config, consoleFile, args })
956
+ for (const b of bad) failed.push(`${node.id} (${locale}) ${b}`)
957
+ } catch (error) {
958
+ failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
959
+ say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
822
960
  }
823
- if (!['playwright', 'script'].includes(node.execution.strategy) || !node.execution.script) { pending.push(`${node.id} (${locale}): ${node.execution.strategy} — ${node.execution.instructions.split('\n')[0]}`); continue }
824
- const streakKey = `${node.session || ''}|${locale}`
825
- if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale})`); continue }
961
+ continue
962
+ }
963
+ if (node.kind === 'email' && (node.mail?.to || sessionInfo(node.session)?.mailbox)) {
964
+ // an e-mail node with a recipient is walked like a step: wait for the message and capture it
826
965
  try {
827
- const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
828
- // the automatic capture runs unless this script invocation captured the step itself: a crop is an
829
- // attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
830
- let captured = false
831
- const capture = (opts = {}) => { captured = true; return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile: conn && !sess ? consoleLog(root, config.output, env.name) : null, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {} }).then((r) => { prefetch?.add(r, run.dir); return r }) }
832
- await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
833
- if (!captured) await capture()
834
- say(`✓ ${node.id} ${locale}`)
835
- streak.ok(streakKey)
966
+ const to = expandRun(node.mail?.to || sessionInfo(node.session).mailbox, run.meta.short)
967
+ const results = await captureMail({ config, root, run, env, node, locale, to, subject: localized(node.mail?.subject, locale), timeoutMs: 45000 })
968
+ say(`✓ ${node.id} ${locale}: "${results.subject}"`)
836
969
  } catch (error) {
837
970
  failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
838
971
  say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
839
- if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
840
972
  }
973
+ continue
841
974
  }
842
- } finally { await page.close().catch(() => {}) }
843
- }
975
+ if (!['playwright', 'script'].includes(node.execution.strategy) || !node.execution.script) { pending.push(`${node.id} (${locale}): ${node.execution.strategy} — ${node.execution.instructions.split('\n')[0]}`); continue }
976
+ const streakKey = `${node.session || ''}|${locale}`
977
+ if (streak.stopped(streakKey)) { skipped.push(`${node.id} (${locale})`); continue }
978
+ const scriptFile = resolve(scenarioDir, node.execution.script)
979
+ // one invocation of the step script; the automatic capture runs unless the script captured the step itself:
980
+ // a crop is an attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
981
+ const runStep = async (mod) => {
982
+ let captured = false
983
+ const capture = (opts = {}) => { captured = true; return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); return r }) }
984
+ await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
985
+ if (!captured) await capture()
986
+ }
987
+ try {
988
+ await runStep(await import(pathToFileURL(scriptFile).href))
989
+ say(`✓ ${node.id} ${locale}`)
990
+ streak.ok(streakKey)
991
+ } catch (error) {
992
+ // a selector the page no longer has: propose one from the page (#13); --heal applies it once the step passes
993
+ const healed = args['no-heal'] ? null : await healStep({ error, page: sPage, scriptFile, root, output: config.output, run, node, locale, apply: !!args.heal, rerun: runStep }).catch(() => null)
994
+ if (healed?.applied) {
995
+ say(`✓ ${node.id} ${locale} (healed)\n${formatHeal(healed, { stepArg: node.id })}`)
996
+ streak.ok(streakKey)
997
+ continue
998
+ }
999
+ failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
1000
+ say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
1001
+ if (healed) { say(formatHeal(healed, { stepArg: node.id })); if (healed.patch) proposals.push(node.id) }
1002
+ if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
1003
+ }
1004
+ }
1005
+ } finally { await page.close().catch(() => {}) }
1006
+ return { pending, failed, skipped }
1007
+ }
1008
+ let results
1009
+ try {
1010
+ if (parallel) say(`Walking ${locales.length} locales, ${concurrency} at a time (each in its own browser context); output in locale order.`)
1011
+ results = await inOrder(locales, parallel ? concurrency : 1, (locale) => walkLocale(locale).catch((error) => ({ pending: [], failed: [`${locale}: ${error.message.split('\n')[0]}`], skipped: [] })))
844
1012
  } finally {
845
1013
  for (const { context } of sessionContexts.values()) await context.close().catch(() => {})
1014
+ for (const c of ownContexts) await c.close().catch(() => {})
846
1015
  if (conn) await conn.browser.close() // over CDP this only disconnects; the shared browser keeps running
847
1016
  if (ephemeral) await ephemeral.close()
848
1017
  await releaseLock?.()
849
1018
  }
1019
+ const pending = results.flatMap((r) => r.pending); const failed = results.flatMap((r) => r.failed); const skipped = results.flatMap((r) => r.skipped)
850
1020
  if (pending.length) say(`\nSteps for an agent or a person (capture them with kaloko capture --step <id>):\n - ${[...new Set(pending)].join('\n - ')}`)
851
1021
  if (failed.length) { say(`\nSteps that failed (fix and rerun with --steps):\n - ${failed.join('\n - ')}`); process.exitCode = 1 }
1022
+ if (proposals.length) say(`\nSelector proposals above: review the patch, then kaloko walk --steps ${[...new Set(proposals)].join(',')} --heal applies it once the step passes.`)
852
1023
  if (skipped.length) { say(`\nSteps not walked after repeated failures (rerun with --steps once the cause is fixed):\n - ${skipped.join('\n - ')}`); process.exitCode = 1 }
853
1024
  if (prefetch) await walkVerdicts({ prefetch, run, scenario, config, root })
854
1025
  },
855
1026
 
1027
+ async affected (args) {
1028
+ const { config, root } = await context(args)
1029
+ let targets = []
1030
+ if (typeof args.scenario === 'string') targets = [resolve(args.scenario)]
1031
+ else {
1032
+ try { const r = await resolveRun({ root, output: config.output, run: args.run }); targets = [resolve(root, r.meta.scenario_file)] } catch {}
1033
+ if (!targets.length || args.all) targets = await scenarioFiles(config, root)
1034
+ }
1035
+ if (!targets.length) throw new UserError('No scenario: pass --scenario <file>, start a run, or list scenarios in the config.')
1036
+ const out = []
1037
+ for (const file of targets) {
1038
+ const scenario = await loadScenario(file)
1039
+ const r = await affectedSelection(args, { config, root, scenario, scenarioFile: file })
1040
+ out.push({ scenario: scenario.id, file: relative(root, file), base: r.base, changed: r.files, runs: r.runs, ...r.sel, walk: selectedIds(scenario, r.sel) })
1041
+ if (!args.json) say(formatAffected(r.sel, { scenario, base: r.base, changed: r.files.length }) + '\n')
1042
+ }
1043
+ if (args.json) say(JSON.stringify(targets.length === 1 && typeof args.scenario === 'string' ? out[0] : out, null, 2))
1044
+ else if (!args.check) say(`Walk them with: kaloko walk --affected${args.base ? ` --base ${args.base}` : ''} (after kaloko start)`)
1045
+ // --check: exit code 1 when nothing is affected (CI skips the scenario)
1046
+ if (args.check && !out.some((o) => o.walk.length && !o.none)) process.exitCode = 1
1047
+ },
1048
+
1049
+ /**
1050
+ * `kaloko import <path>` (#11): screenshots, recordings and traces from agents and tools as a run: into the current
1051
+ * run, a new run of --scenario, or an ad-hoc flow made from the screens themselves.
1052
+ */
1053
+ async import (args) {
1054
+ const from = args._[0]
1055
+ if (!from) throw new UserError('Usage: kaloko import <folder | playwright-report/ | results.json | trace.zip | walkthrough.md> [--scenario file | --adhoc] [--env e] [--locale l] [--viewport v] [--map "Title=step,…"]')
1056
+ const { config, root } = await context(args)
1057
+ let source
1058
+ try { source = await readSource(resolve(from), typeof args.format === 'string' ? args.format : 'auto') } catch (e) { throw new UserError(`kaloko import: ${e.message}`) }
1059
+ if (!source.items.length) throw new UserError(`No screenshots found in ${from} (${source.format}). A folder needs .png, .jpg or .webp files, a steps.json or a Markdown walkthrough with images; a Playwright report needs screenshot attachments or a trace.`)
1060
+ say(`Read ${source.items.length} screen(s)${source.videos.length ? ` and ${source.videos.length} recording(s)` : ''} from ${from} (${source.format}${source.tool !== source.format ? `, ${source.tool}` : ''})`)
1061
+ const envName = () => {
1062
+ if (typeof args.env === 'string') return args.env
1063
+ const names = Object.keys(config.environments).filter((n) => config.environments[n].kind !== 'design')
1064
+ for (const n of ['local', ...names]) { try { if (config.environments[n]) { environment(config, n); return n } } catch {} }
1065
+ throw new UserError('Pass --env <name>: no environment of the config resolves here.')
1066
+ }
1067
+ let run = null
1068
+ let scenarioFile = null
1069
+ if (typeof args.scenario === 'string') scenarioFile = resolve(args.scenario)
1070
+ else if (!args.adhoc) { try { run = await resolveRun({ root, output: config.output, run: args.run }) } catch {} }
1071
+ if (!run && !scenarioFile) {
1072
+ // an ad-hoc flow: one manual screen per imported item, in order; read-only, it never touched the environment
1073
+ const scenario = adhocScenario(source, { id: typeof args.id === 'string' ? args.id : undefined, title: typeof args.title === 'string' ? args.title : undefined, locale: typeof args.locale === 'string' ? args.locale : 'en' })
1074
+ scenario.readonly = true
1075
+ const problems = validateScenario(JSON.parse(JSON.stringify(scenario)))
1076
+ if (problems.length) throw new UserError(`The ad-hoc flow did not validate:\n - ${problems.join('\n - ')}`)
1077
+ scenarioFile = resolve(root, typeof args.out === 'string' ? args.out : join('qa', 'flows', `${slug(scenario.id)}.yml`))
1078
+ if (existsSync(scenarioFile) && !args.force) throw new UserError(`${relative(process.cwd(), scenarioFile)} exists; pass --out <file> or --force.`)
1079
+ await mkdir(dirname(scenarioFile), { recursive: true })
1080
+ await writeFile(scenarioFile, `# Ad-hoc flow from kaloko import ${from}: rename steps, add purposes and criteria, or map the screens onto a real scenario.\n${scenarioYaml(scenario)}`)
1081
+ say(`Wrote the ad-hoc flow ${relative(process.cwd(), scenarioFile)} (${scenario.nodes.length} steps)`)
1082
+ }
1083
+ if (!run) {
1084
+ await commands.start({ scenario: scenarioFile, env: envName(), locales: args.locales, viewports: args.viewports, executor: args.executor || `import:${source.tool}`, config: args.config, _: [] })
1085
+ run = await resolveRun({ root, output: config.output })
1086
+ } else say(`Importing into run ${run.meta.id} (${run.meta.scenario.id}); --adhoc or --scenario starts a new one`)
1087
+ const scenario = await loadScenario(resolve(root, run.meta.scenario_file))
1088
+ const map = Object.fromEntries((list(args.map) || []).map((p) => p.split('=')).filter((p) => p.length === 2 && p[0] && p[1]).map(([a, b]) => [a.trim(), b.trim()]))
1089
+ const locale = typeof args.locale === 'string' && run.meta.scope.locales.includes(args.locale) ? args.locale : run.meta.scope.locales[0]
1090
+ const enc = await launchEphemeral(config).catch(() => null)
1091
+ const encoder = enc ? await enc.newPage().catch(() => null) : null
1092
+ const used = new Set(); const unmatched = []; let captured = 0; let attached = 0
1093
+ try {
1094
+ for (const item of source.items) {
1095
+ const hit = stepFor(item, scenario.nodes, map)
1096
+ if (!hit) { unmatched.push(item.title); say(`? ${item.title}: no step matches`); continue }
1097
+ const loc = item.locale && run.meta.scope.locales.includes(item.locale) ? item.locale : locale
1098
+ const vp = viewportFor(item.size, run.meta.scope.viewports, item.viewport || (typeof args.viewport === 'string' ? args.viewport : null))
1099
+ const key = `${hit.node.id}|${loc}|${vp}`
1100
+ if (used.has(key)) { await attachImported({ run, node: hit.node, locale: loc, item }); attached++; say(`+ ${item.title} → attached to ${hit.node.id}`); continue }
1101
+ used.add(key)
1102
+ await writeImported({ run, node: hit.node, locale: loc, viewport: vp, item, tool: source.tool, encoder })
1103
+ captured++
1104
+ say(`✓ ${item.title} → ${hit.node.id} ${loc}/${vp} (${hit.how})`)
1105
+ }
1106
+ } finally { await enc?.close().catch(() => {}) }
1107
+ const kept = await keepVideos(run, source.videos)
1108
+ say(`\nImported ${captured} screen(s)${attached ? `, ${attached} more as attachments` : ''} into ${run.meta.id}.${encoder ? '' : ' (No Chrome: images kept as they are, without thumbnails.)'}`)
1109
+ if (unmatched.length) say(`Not matched to a step (${unmatched.length}): name the step in steps.json ("step": "<id>"), or pass --map "${unmatched[0]}=<step id>". Steps: ${scenario.nodes.filter((n) => n.kind !== 'decision').map((n) => n.id).join(', ')}`)
1110
+ if (kept.length) say(`Recordings kept in the run folder, not uploaded (the canvas does not play them yet): ${kept.join(', ')}`)
1111
+ say('Next: kaloko evaluate, then kaloko preview (local canvas) or kaloko share. Imported screens have no page to check: their deterministic criteria wait for a walk, semantic ones for a person.')
1112
+ },
1113
+
1114
+ /**
1115
+ * `kaloko ci github|gitlab` writes the workflow that walks pull request previews (#15); `kaloko ci preview-url`
1116
+ * runs inside it and finds the preview deployment, the pull request and its base.
1117
+ */
1118
+ async ci (args) {
1119
+ const action = args._[0]
1120
+ if (action === 'preview-url') {
1121
+ const err = (m) => console.error(m)
1122
+ const preview = await findPreview({ wait: Number(args.wait) || 0, say: err })
1123
+ const pr = await pullRequestOf().catch(() => null)
1124
+ if (!preview) throw new UserError('No preview URL found. Set KALOKO_PREVIEW_URL, or let Vercel, Netlify or Cloudflare Pages report deployments to GitHub (the workflow needs deployments: read).')
1125
+ const lines = envLines({ preview, pr })
1126
+ if (args['github-env']) {
1127
+ if (!process.env.GITHUB_ENV) throw new UserError('--github-env works inside GitHub Actions (GITHUB_ENV is not set).')
1128
+ await writeEnvFile(process.env.GITHUB_ENV, lines)
1129
+ }
1130
+ if (typeof args.dotenv === 'string') await writeEnvFile(resolve(args.dotenv), lines)
1131
+ err(`Preview ${preview.url} (from ${preview.source})${pr?.number ? `, pull request #${pr.number}${pr.base ? ` into ${pr.base}` : ''}` : ''}`)
1132
+ console.log(preview.url)
1133
+ return
1134
+ }
1135
+ if (!['github', 'gitlab'].includes(action)) throw new UserError('Usage: kaloko ci github|gitlab [--scenario a.yml,b.yml] [--env preview] [--trigger deployment|pull_request] [--no-affected] [--out file] [--print] [--force]\n kaloko ci preview-url [--wait 600] [--github-env] [--dotenv file]')
1136
+ const { config, root, file } = await context(args)
1137
+ const envName = typeof args.env === 'string' ? args.env : 'preview'
1138
+ const all = (await scenarioFiles(config, root)).map((f) => relative(root, f))
1139
+ const scenarios = list(args.scenario) || all
1140
+ if (!scenarios.length) throw new UserError('No scenarios to walk: pass --scenario <file>, or list scenario globs in the config.')
1141
+ // the environment the workflow walks: base_url from KALOKO_PREVIEW_URL, which preview-url sets
1142
+ if (!config.environments[envName]) {
1143
+ const text = await readFile(file, 'utf8')
1144
+ const next = addPreviewEnvironment(text, envName)
1145
+ if (next && !args.print) {
1146
+ const problems = (await import('./lib/schema.mjs')).schemaErrors('config.v1', yaml.load(next))
1147
+ if (problems.length) throw new UserError(`Adding ${envName} would make ${CONFIG_FILE} invalid: ${problems.join('; ')}`)
1148
+ await writeFile(file, next)
1149
+ say(`Added the environment ${envName} to ${relative(process.cwd(), file) || CONFIG_FILE}: base_url \${KALOKO_PREVIEW_URL}, set by kaloko ci preview-url in CI.`)
1150
+ } else say(`Add to environments: in ${CONFIG_FILE}:\n ${envName}:\n base_url: \${KALOKO_PREVIEW_URL}`)
1151
+ }
1152
+ for (const f of scenarios) {
1153
+ let sc = null
1154
+ try { sc = yaml.load(await readFile(resolve(root, f), 'utf8')) } catch { throw new UserError(`${f} cannot be read.`) }
1155
+ if (sc?.environments && !sc.environments.includes(envName)) say(`! ${f} allows environments [${sc.environments.join(', ')}]: add ${envName}, or kaloko start refuses it in CI.`)
1156
+ }
1157
+ let pkg = null
1158
+ try { pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')) } catch {}
1159
+ const pm = existsSync(join(root, 'pnpm-lock.yaml')) ? 'pnpm' : existsSync(join(root, 'yarn.lock')) ? 'yarn' : existsSync(join(root, 'bun.lockb')) || existsSync(join(root, 'bun.lock')) ? 'bun' : 'npm'
1160
+ const dependency = !!(pkg && ({ ...pkg.dependencies, ...pkg.devDependencies }).kaloko)
1161
+ const version = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8')).version
1162
+ const opts = { scenarios, env: envName, affected: !args['no-affected'], pm, dependency, version: `^${version}`, trigger: args.trigger === 'pull_request' ? 'pull_request' : 'deployment' }
1163
+ const content = action === 'github' ? githubWorkflow(opts) : gitlabJob(opts)
1164
+ if (args.print) { say(content); return }
1165
+ const out = resolve(root, typeof args.out === 'string' ? args.out : action === 'github' ? '.github/workflows/kaloko.yml' : 'kaloko.gitlab-ci.yml')
1166
+ if (existsSync(out) && !args.force) throw new UserError(`${relative(process.cwd(), out)} exists; pass --force to replace it (or --print to compare).`)
1167
+ await mkdir(dirname(out), { recursive: true })
1168
+ await writeFile(out, content)
1169
+ say(`Wrote ${relative(process.cwd(), out)}: ${scenarios.length} scenario(s) on ${envName}${opts.affected ? ', only what each pull request affects' : ''}${action === 'github' ? `, ${opts.trigger === 'deployment' ? 'when a preview deployment succeeds' : 'on every pull request'}` : ''}.`)
1170
+ say(action === 'github'
1171
+ ? 'Next:\n add the repository secret KALOKO_TOKEN: a token with the tester role and the upload scope (Kaloko → Settings → API tokens; on Enterprise, a service account token)\n commit the workflow; the first preview deployment of a pull request walks it and comments on the PR\n Vercel deployment protection: add VERCEL_AUTOMATION_BYPASS_SECRET and uncomment access.headers in the preview environment'
1172
+ : 'Next:\n include it from .gitlab-ci.yml: include: { local: kaloko.gitlab-ci.yml }\n add the masked CI/CD variables KALOKO_TOKEN and KALOKO_GITLAB_TOKEN (api scope, for the merge request comment)\n make the job wait for your review app deploy (needs:), or set KALOKO_PREVIEW_URL in its variables')
1173
+ },
1174
+
856
1175
  async auth (args) {
857
1176
  const { config, root } = await context(args)
858
1177
  const action = args._[0]
@@ -949,6 +1268,7 @@ const commands = {
949
1268
  const report = await evaluateRun({ run, scenario, config, root })
950
1269
  const via = report.via.qawalk ? ` via Kaloko${report.hosted?.limit != null ? ` (${report.hosted.questions} of ${report.hosted.limit} questions this month)` : ''}` : report.via['own-key'] ? ' with your key' : ''
951
1270
  say(`Evaluated ${report.evaluated} capture(s); semantic evaluator ${report.model}${via}: ${report.semantic_calls} call(s), ${report.semantic_cached} cached.`)
1271
+ if (report.vision?.asked) say(`Vision evaluator ${report.vision.model}: ${report.vision.asked} capture(s) with flagged or uncertain criteria, ${report.vision.calls} call(s), ${report.vision.cached} cached${report.vision.skipped.length ? `; not asked: ${[...new Set(report.vision.skipped.map((x) => x.split(': ').slice(1).join(': ')))].slice(0, 2).join('; ')}` : ''}.`)
952
1272
  if (report.skipped.length) {
953
1273
  // one line per reason: "12 captures: TYPESAFE_API_KEY is not set …"
954
1274
  const byReason = new Map()
@@ -964,9 +1284,15 @@ const commands = {
964
1284
  const file = await writePreview({ run, manifest })
965
1285
  const s = manifest.summary
966
1286
  say(`${manifest.id}: ${s.pass} pass · ${s.partial} partial · ${s.fail} fail · ${s.skip} not checked\nPreview: ${file}`)
1287
+ const why = attentionLines(manifest); if (why.length) say(`AI verdicts to act on (score and why):\n${why.join('\n')}`)
967
1288
  return file
968
1289
  },
969
1290
 
1291
+ async stability (args) {
1292
+ const { config, root, run, scenario } = await runContext(args)
1293
+ await stabilityCmd({ root, config, run, scenario, args, say })
1294
+ },
1295
+
970
1296
  async preview (args) {
971
1297
  const file = await commands.build(args)
972
1298
  say(pathToFileURL(file).href)
@@ -989,7 +1315,7 @@ const commands = {
989
1315
  await writePreview({ run, manifest })
990
1316
  const files = new Set()
991
1317
  for (const n of manifest.nodes) {
992
- for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree].filter(Boolean).forEach((p) => files.add(p))
1318
+ for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree, c.spec].filter(Boolean).forEach((p) => files.add(p))
993
1319
  for (const a of n.attachments || []) files.add(a.file)
994
1320
  // stack steps: the items index and every page's files
995
1321
  if (n.stack?.items) {
@@ -999,39 +1325,22 @@ const commands = {
999
1325
  }
1000
1326
  }
1001
1327
  const days = Number(args.expires) || config.share?.expires_days || 30
1002
- const { stat } = await import('node:fs/promises')
1003
- let total = 0
1004
- for (const p of files) total += (await stat(join(run.dir, p))).size
1328
+ // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
1329
+ const entries = await hashFiles(run.dir, [...files])
1330
+ const missing = await missingHashes(client, config.org, entries.map((e) => e.hash))
1331
+ const total = entries.reduce((n, e) => n + e.bytes, 0)
1332
+ const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.bytes])).values()].reduce((n, b) => n + b, 0) : total
1333
+ const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
1005
1334
  try {
1006
1335
  const u = await client.request('GET', `/orgs/${config.org}/usage`)
1007
1336
  if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}. Capture fewer viewports or locales, or ask an admin for a higher limit.`)
1008
- if (u.usage.bytes + total > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(total)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
1009
- say(`Uploading ${files.size} files, ${fmtBytes(total)} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
1337
+ if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
1338
+ say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
1010
1339
  } catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
1011
1340
  const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
1012
- // text files travel and rest gzip-compressed; the service serves them with Content-Encoding: gzip
1013
- const body = async (p) => { const raw = await readFile(join(run.dir, p)); const gz = /\.(html|json)$/.test(p) && raw.length > 512; return { data: gz ? gzipSync(raw, { level: 6 }) : raw, gz } }
1014
- // the file list goes first: the service checks limits once and each upload is then a plain write
1015
- // (an older service without the plan answers 404/405 and every upload checks itself, as before)
1016
- // screenshots name their SHA-256: the service checks it on arrival and compares unchanged steps with it
1017
- if (files.size) {
1018
- const plan = []
1019
- for (const p of files) { const { data } = await body(p); plan.push({ path: p, bytes: data.length, ...(/\.(png|webp|jpe?g)$/i.test(p) ? { sha256: createHash('sha256').update(data).digest('hex') } : {}) }) }
1020
- await client.request('POST', `/orgs/${config.org}/runs/${manifest.id}/files`, { files: plan }).catch((e) => { if (!/HTTP (404|405)/.test(e.message)) throw e })
1021
- }
1022
- const queue = [...files]
1023
- let done = 0
1024
- const worker = async () => {
1025
- while (queue.length) {
1026
- const p = queue.shift()
1027
- const { data, gz } = await body(p)
1028
- await client.upload(`/orgs/${config.org}/runs/${manifest.id}/files/${p.split('/').map(encodeURIComponent).join('/')}`, data, p, gz ? { 'Content-Encoding': 'gzip' } : {})
1029
- done++
1030
- if (done % 10 === 0 || done === files.size) process.stdout.write(`\r uploaded ${done}/${files.size}`)
1031
- }
1032
- }
1033
- await Promise.all(Array.from({ length: files.size > 200 ? 12 : 6 }, worker))
1034
- if (files.size) process.stdout.write('\n')
1341
+ await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})
1342
+ const up = await uploadRunFiles({ client, org: config.org, runId: manifest.id, dir: run.dir, entries, missing, say })
1343
+ if (up.mode === 'blobs') say(` ${up.uploaded} new file(s) sent (${fmtBytes(up.bytes)}), ${up.reused} unchanged file(s) reused`)
1035
1344
  // experimental HTML views (#42): their bundles travel as blobs and attach to the run as revisions
1036
1345
  const viewIds = new Set(manifest.nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((vps) => Object.values(vps).map((c) => c.view?.revision).filter(Boolean))))
1037
1346
  if (viewIds.size) {
@@ -1046,6 +1355,7 @@ const commands = {
1046
1355
  } catch (e) { say(`HTML views not attached: ${e.message.split('\n')[0]}`) }
1047
1356
  }
1048
1357
  const fin = await client.request('POST', `/orgs/${config.org}/runs/${manifest.id}/finalize`)
1358
+ await rm(join(run.dir, PROGRESS_FILE), { force: true })
1049
1359
  const meta = await readJson(join(run.dir, 'meta.json'))
1050
1360
  meta.shared = { url: fin.url, at: new Date().toISOString(), expires_days: created.expires_days }
1051
1361
  await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
@@ -1097,6 +1407,25 @@ const commands = {
1097
1407
  await historyCmd({ config, scenarioId, client: service(config), args, say })
1098
1408
  },
1099
1409
 
1410
+ async spec (args) {
1411
+ const { config, root } = await context(args)
1412
+ let run = null
1413
+ try { run = await resolveRun({ root, output: config.output, run: args.run }) } catch {}
1414
+ const scenario = run ? await loadScenario(resolve(root, run.meta.scenario_file)).catch(() => null) : typeof args.scenario === 'string' && /\.ya?ml$/.test(args.scenario) ? await loadScenario(resolve(args.scenario)) : null
1415
+ const scenarioId = await scenarioIdOf(args, config, root)
1416
+ let client = null
1417
+ if (!args.local) { try { client = service(config) } catch (e) { if (!run) throw e } }
1418
+ await specCmd({ config, client, run, scenario, scenarioId, args, say })
1419
+ },
1420
+
1421
+ async drift (args) {
1422
+ const { config, root } = await context(args)
1423
+ let run = null
1424
+ try { run = await resolveRun({ root, output: config.output, run: args.run }) } catch {}
1425
+ const scenarioId = await scenarioIdOf(args, config, root)
1426
+ await driftCmd({ config, client: service(config), run, scenarioId, args, say })
1427
+ },
1428
+
1100
1429
  async reference (args) {
1101
1430
  const { config, root } = await context(args)
1102
1431
  const scenarioId = await scenarioIdOf(args, config, root)