kaloko 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.mjs CHANGED
@@ -20,7 +20,8 @@ import { evaluateRun, semanticPrefetch } from './lib/evaluate.mjs'
20
20
  import { buildManifest, writePreview, attentionLines } from './lib/build.mjs'
21
21
  import { stabilityCmd, pullDrawn } from './lib/stability.mjs'
22
22
  import { signoffCmd } from './lib/signoff.mjs'
23
- import { localized, readJson, UserError, UsageError, globToRegExp, failureStreak, expiryNote, SUPPORT_LINE, SUPPORT_EMAIL, NO_ORG } from './lib/util.mjs'
23
+ import { localized, readJson, UserError, UsageError, globToRegExp, failureStreak, expiryNote, SUPPORT_LINE, SUPPORT_EMAIL, NO_ORG, EXIT, parseDuration } from './lib/util.mjs'
24
+ import { noteLegacy, legacyHint, resetLegacy, legacyInUse } from './lib/legacy.mjs'
24
25
  import { packageRoot } from './lib/schema.mjs'
25
26
  import { compareRuns, formatComparison } from './lib/compare.mjs'
26
27
  import { doctor, formatDoctor } from './lib/doctor.mjs'
@@ -71,13 +72,19 @@ const COMMON_HELP = `Common options (every command)
71
72
  --run <id|dir> The run to act on (default: the current run, or KALOKO_RUN)
72
73
  --config <file> Another kaloko.config.yml
73
74
  --org <slug> Another organization than org: in the config
74
- --json Machine-readable output, where a command prints data
75
- --help, -h Help for one command, like kaloko help <command>
75
+ --json Machine-readable output, where a command prints data: stdout holds only that
76
+ JSON, progress and notes go to stderr, and an error is one line on stderr:
77
+ {"error":{"message":"…","code":"…","next":"…"}} with the exit code below
78
+ --help, -h Help for one command, like kaloko help <command> (kaloko help export run: one
79
+ sub-command)
76
80
  --version, -v The version of kaloko (also kaloko version)
77
81
  KALOKO_AGENT=<name> Labels writes by an agent
82
+ Durations (--timeout, --wait, --expires, --older-than) take a number in the option's unit or one with s, m, h or d:
83
+ 90s, 10m, 2h, 14d
78
84
 
79
85
  Exit codes: 0 done · 1 checks found problems or the command failed · 2 usage error (unknown command or option,
80
- missing argument)
86
+ missing argument) · 3 the Kaloko service cannot be reached (no connection; an HTTP error answer is 1) · 4 no token,
87
+ or an invalid, expired or revoked one (kaloko login)
81
88
  `
82
89
 
83
90
  const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
@@ -103,9 +110,10 @@ Project
103
110
  Write .github/workflows/kaloko-release.yml: when release-please, changesets or
104
111
  semantic-release publishes a GitHub release, kaloko release records it on Kaloko
105
112
  (--snippet prints the step to add after the tool's own step instead)
106
- ci preview-url [--wait 600] [--github-env] [--dotenv file]
113
+ ci preview-url [--wait 10m] [--github-env] [--dotenv file] [--json]
107
114
  In CI: the preview URL (KALOKO_PREVIEW_URL, the deployment event, provider
108
- variables or the commit's GitHub deployments), the pull request and its base
115
+ variables or the commit's GitHub deployments), the pull request and its base;
116
+ --wait polls that long for the deployment (seconds, or 10m)
109
117
  login [--org <slug>] [--url <service>] [--print] [--no-open]
110
118
  Sign this machine in through the browser: approve the code on kaloko.app, choose
111
119
  the organization and what the token may do; the token (at most 30 days) goes to
@@ -114,10 +122,10 @@ Project
114
122
  and app tools; --fix installs missing Playwright browsers, creates the run and
115
123
  scenario folders, writes org: from the token and asks macOS for the desktop
116
124
  capture permissions
117
- validate <scenario.yml>... [--coverage [--source ACCEPTANCE.md]]
125
+ validate <scenario.yml>... [--coverage [--source ACCEPTANCE.md]] [--json]
118
126
  Validate scenarios (schema + references); --coverage compares criteria with the plan
119
127
  With a token it also warns about scenarios the service still lists whose file is gone (--offline skips it)
120
- scenarios List scenarios matched by the config
128
+ scenarios [--json] List scenarios matched by the config
121
129
 
122
130
  Run
123
131
  start --scenario <file> --env <name> [--locales cs,en] [--viewports desktop,mobile] [--executor <name>]
@@ -135,7 +143,7 @@ Run
135
143
  Design environments: the token modes to render (default every mode of the set)
136
144
  [--pii redact|pseudonymize|off] Hide personal data in every capture (placeholders or consistent fake values);
137
145
  default from the environment's or the scenario's pii
138
- browser start|stop|status [--env e] [--headed] One shared browser per environment (port = browser.port + index)
146
+ browser start|stop|status [--env e] [--headed] [--json] One shared browser per environment (port = browser.port + index)
139
147
  Shared Chromium with CDP (agents, scripts and people drive it)
140
148
  browser install [--with-deps] [--browsers chromium,webkit,firefox]
141
149
  Download the engines of kaloko's Playwright (default Chromium; CI:
@@ -152,7 +160,7 @@ Run
152
160
  An app screen taken by any tool (Maestro, Appium, XCUITest, a person) with its
153
161
  UI dump (uiautomator/Appium XML, idb or Maestro JSON); app scenarios without
154
162
  --image capture the current screen of the environment's app
155
- mail --step <id> [--locale l] [--to <addr>] [--subject <text>] [--timeout 60]
163
+ mail --step <id> [--locale l] [--to <addr>] [--subject <text>] [--timeout 60s]
156
164
  [--since <ISO date> | --any-time] [--id <mailbox message id>]
157
165
  Only messages received since the run started, unless --since/--any-time; --id takes one exact message
158
166
  Capture an e-mail from the environment's mailbox (mail.provider) as an email step
@@ -161,7 +169,7 @@ Run
161
169
  auth save --env <e> --account <key> [--url <url>]
162
170
  A person signs in once (SSO, MFA) in a visible browser; the session is saved
163
171
  for that account and walk reuses it (never uploaded)
164
- auth list Saved sessions per environment and account, with their age
172
+ auth list [--json] Saved sessions per environment and account, with their age
165
173
  walk [--steps a,b] [--locales l] [--keep-going] [--ephemeral] [--browsers webkit]
166
174
  Run playwright steps (execution.script) and capture them; a session stops
167
175
  after 3 steps in a row fail with the same error unless --keep-going
@@ -189,10 +197,11 @@ Run
189
197
  per locale with a chapter per step (flow), or only steps that fail (failed);
190
198
  also capture.record in the config or scenario, record: true on a step.
191
199
  A trace (actions, requests, console) is kept per step unless --no-trace
192
- affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--check] [--offline]
200
+ affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--fail-if-none] [--offline]
193
201
  Which steps changed files reach, and why: framework route files against step paths
194
202
  and URLs captured earlier, imports, global files, i18n messages, step scripts and
195
- affects: globs; --check exits 1 when nothing is affected
203
+ affects: globs; --fail-if-none exits 1 when nothing is affected (--check, its
204
+ older name, still works)
196
205
  import <folder | playwright-report/ | results.json | trace.zip | walkthrough.md> [--scenario file | --adhoc]
197
206
  [--env e] [--locale l] [--viewport v] [--map "Title=step,…"] [--format f] [--out file]
198
207
  Screenshots, recordings and traces from agents and tools as a run: into the
@@ -201,18 +210,19 @@ Run
201
210
  Scenarios may declare "sessions" (people in the flow); walk keeps one browser context per session and locale
202
211
 
203
212
  Results
204
- evaluate Deterministic checks + semantic evaluator (default JEV) for all captures
213
+ evaluate [--json] Deterministic checks + semantic evaluator (default JEV) for all captures
205
214
  evaluate --recheck [--steps a,b] After a scenario change: re-run deterministic checks on the stored HTML and ask
206
215
  the semantic questions again on the stored captures (no walk);
207
216
  console, overflow and performance keep their live result
208
- build Write run.json and the local preview (index.html)
217
+ build [--json] Write run.json and the local preview (index.html)
209
218
  video export [--format mp4|webm|gif] [--locale l] [--steps a,b] [--lang en|cs] [--out file]
210
219
  The run's recordings in flow order with a title card per step (title, verdict);
211
220
  shared with the run on Business and up
212
221
  video list | trace --step <id> | tools
213
222
  Recordings and traces of the run; a step's trace as a timeline; ffmpeg, adb, simctl
214
223
  preview Build and print the preview path
215
- share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
224
+ share [--expires 30d] [--pr] [--json]
225
+ Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
216
226
  hash, so unchanged captures are not sent again and a broken-off share resumes;
217
227
  --pr adds or updates one comment on the branch's pull request via gh (KALOKO_PR in
218
228
  CI), or a note on the GitLab merge request (KALOKO_GITLAB_TOKEN)
@@ -229,12 +239,13 @@ Results
229
239
  comment --reply|--resolve|--reopen|--edit|--delete <id> [--note "…"] [--flow]
230
240
  Comment, answer and close feedback from the CLI (resolve with "Fixed in <commit>")
231
241
  projects [--json] Projects of the organization (use the slug as project: in a scenario or config)
232
- runs [--scenario id] [--env e] [--all] [--json]
242
+ runs [--scenario id] [--env e] [--all] [--limit N] [--cursor c] [--json]
233
243
  Shared runs: the newest per scenario (with "+ N older"); --all or --scenario lists every
234
- run (the newest 300; --scenario and --env narrow the list)
244
+ run: --all to the last one page by page, otherwise the newest 300; --limit N lists at
245
+ most N, --cursor c goes on where a list stopped (it prints the cursor)
235
246
  trash <run-id>... | restore <run-id>
236
247
  Move runs to the trash (uploader or admin) and back; restorable until purge
237
- prune [--scenario id] [--env e] [--keep 3] [--older-than 14d] [--dry-run | --yes] [--json]
248
+ prune [--scenario id] [--env e] [--keep 3] [--older-than 14d|48h] [--dry-run | --yes] [--json]
238
249
  Keep the newest runs per scenario and environment plus kept, accepted, baseline,
239
250
  referenced and commented runs; lists what would go to the trash (--dry-run, the
240
251
  default) unless --yes
@@ -246,11 +257,14 @@ Results
246
257
  Semantic evaluator vs human labels: precision/recall per criterion at the
247
258
  configured thresholds and suggested thresholds (labels from a golden file or
248
259
  from reviewers' criterion verdicts in the service)
249
- export [--run id] [--out file.zip] Download a shared run as a zip before it expires: run.json, every file, the live
260
+ export run [--run id] [--out file.zip]
261
+ Download a shared run as a zip before it expires: run.json, every file, the live
250
262
  HTML of design steps and index.html, the canvas that opens offline from disk
251
- export --scenario <id> [--accepted] [--out dir] [--locales l] [--viewports v]
263
+ (also kaloko export --run id)
264
+ export steps --scenario <id> [--accepted] [--out dir] [--locales l] [--viewports v]
252
265
  Newest capture of every step as steps/<step>/<locale>/<viewport>.<ext> plus
253
266
  index.json (documentation layer, Business plan); unchanged files are skipped
267
+ (also kaloko export --scenario <id>)
254
268
  public [--run id] [--off] Make a shared run readable without sign-in (admin; testers may refresh an already public scenario)
255
269
  stability [--last 4 | --runs a,b] [--same-commit] [--apply] [--json]
256
270
  Regions that change on every run or move while the page is idle (clocks, ads,
@@ -286,6 +300,9 @@ Hosted runs (read-only scenarios on Kaloko's browsers, Business plan)
286
300
  run as you: public pages, HTTP Basic from the environment, or a saved session
287
301
  (--account). Steps that click or type are left out; they need the full walk
288
302
  schedule list | run <id> | pause <id> | resume <id> | remove <id> [--json]
303
+ schedule hook <id> [--remove] [--json]
304
+ A URL and secret that start the hosted run from a deploy (Vercel, Netlify) or
305
+ any CI job; the secret is shown once, a second call replaces it
289
306
  schedule export github --scenario <file> --env <name> [--every day|week] [--at 5] [--out file] [--print]
290
307
  A GitHub Actions workflow that walks the scenario on a schedule with the full CLI
291
308
 
@@ -321,12 +338,13 @@ Docs (Markdown in docs/ with step references; Diátaxis folders; translations as
321
338
  text was published), missing or outdated translations and user-visible changelog
322
339
  items without a picture; exit 1 on references or stale sections (--strict: on
323
340
  anything)
324
- export --format md|docusaurus|vitepress|mkdocs|html|pdf|json [--product p] [--version v] [--lang l]
341
+ export docs [--format md|docusaurus|vitepress|mkdocs|html|pdf|json] [--product p] [--version v] [--lang l]
325
342
  [--what all|docs|changelog] [--local] [--out dir|file]
326
343
  The docs and changelog with their pictures as files: Markdown with an assets
327
- folder, a Docusaurus, VitePress or MkDocs folder with sidebars, offline HTML, PDF
328
- (printed by the browser) or the changelog as JSON; --local exports the files in
329
- the repository instead of the published ones
344
+ folder (the default), a Docusaurus, VitePress or MkDocs folder with sidebars,
345
+ offline HTML, PDF (printed by the browser) or the changelog as JSON; --local
346
+ exports the files in the repository instead of the published ones (also kaloko
347
+ export --format <f>)
330
348
 
331
349
  Design (environments with kind: design; live HTML steps in a folder)
332
350
  start --scenario <file> --env <design env> A run of the design folder (same scenario as development and QA)
@@ -363,13 +381,15 @@ Design (environments with kind: design; live HTML steps in a folder)
363
381
  The project's token set (DTCG); check lists raw values outside it in the live folder
364
382
  branch list | create <name> [--from vN] [--git <branch>] | merge <name> [--take-branch a,b] [--keep-main c]
365
383
  Branches of the draft; a git-linked branch is what implementation runs of that git branch check against
366
- export --live [--version n|--draft|--run id] [--out design/]
384
+ export live [--version n|--draft|--run id] [--out design/]
367
385
  Live files of a version (default: the design reference) into a folder: the handoff
368
- export --live --views --run <id> [--viewport desktop] [--out design/]
386
+ (also kaloko export --live)
387
+ export live --views --run <id> [--viewport desktop] [--out design/]
369
388
  Fork into a draft: the HTML views of an app run become design steps (experimental)
370
- export --design [--version n|--draft|--run id] [--out dir|file.zip]
389
+ export design [--version n|--draft|--run id] [--out dir|file.zip]
371
390
  A design version in open formats: HTML + assets, tokens.json (DTCG), scenario.yml,
372
- screenshots and an index.html that opens from disk (default: the design reference)
391
+ screenshots and an index.html that opens from disk (default: the design reference;
392
+ also kaloko export --design)
373
393
 
374
394
  ${COMMON_HELP}`
375
395
 
@@ -384,18 +404,19 @@ export function regionText (g) {
384
404
 
385
405
  /** validate: scenarios the service still lists as active whose scenario file no longer exists here (#58). */
386
406
  async function warnRetired (args) {
387
- if (args.offline) return
407
+ if (args.offline) return []
388
408
  let config, client
389
- try { ({ config } = await context(args)); client = service(config) } catch { return } // no project or no token: offline
390
- if (!config?.org) return
409
+ try { ({ config } = await context(args)); client = service(config) } catch { return [] } // no project or no token: offline
410
+ if (!config?.org) return []
391
411
  let list
392
- try { list = await client.request('GET', `/orgs/${config.org}/scenarios?runs=1`) } catch { return }
412
+ try { list = await client.request('GET', `/orgs/${config.org}/scenarios?runs=1`) } catch { return [] }
393
413
  const gone = []
394
414
  for (const s of list || []) {
395
415
  if (s.archived || !s.file || (config.project && s.project && s.project !== config.project)) continue
396
416
  try { await readFile(resolve(s.file)) } catch { gone.push(s) }
397
417
  }
398
- for (const s of gone) say(`! ${s.id}: the service lists it as active, but ${s.file} is gone here (split or renamed?). Archive it: kaloko archive ${s.id}`)
418
+ if (!args.json) for (const s of gone) say(`! ${s.id}: the service lists it as active, but ${s.file} is gone here (split or renamed?). Archive it: kaloko archive ${s.id}`)
419
+ return gone.map((s) => ({ id: s.id, file: s.file }))
399
420
  }
400
421
 
401
422
  /** The end of a walk: evaluate (the semantic answers are mostly cached by now) and say what did not pass. */
@@ -936,6 +957,29 @@ async function captureImage ({ config, run, scenario, node, locale, args }) {
936
957
  }
937
958
 
938
959
  /** walk --affected / kaloko affected: the selection for one scenario (a token adds URLs of the newest shared run). */
960
+ /**
961
+ * `kaloko export run|steps|docs|live|design …`: the sub-command sets the flag form it stands for (export --live,
962
+ * --design, --format, --scenario), which keeps working. A flag of another kind of export is a usage mistake.
963
+ */
964
+ export function exportSubcommand (args) {
965
+ const sub = args._[0]
966
+ if (sub === undefined) return null
967
+ // the flags of the other kinds (live and design exports take --scenario too)
968
+ const clashes = { run: ['live', 'design', 'format', 'scenario'], steps: ['live', 'design', 'format'], docs: ['live', 'design', 'scenario'], live: ['design', 'format'], design: ['live', 'format'] }
969
+ const owner = { live: 'live', design: 'design', format: 'docs', scenario: 'steps' }
970
+ // kaloko export <run-id>: the run named as a word, as people type it
971
+ if (/^[0-9]{4}-[0-9]{2}-[0-9]{2}-[a-z0-9-]+$/.test(sub) && args.run === undefined) { args._.shift(); args.run = sub; return 'run' }
972
+ if (!(sub in clashes)) throw new UsageError(`kaloko export ${sub}: there is no such export. Use run, steps, docs, live or design (kaloko help export).`)
973
+ args._.shift()
974
+ const clash = clashes[sub].find((f) => args[f] !== undefined)
975
+ if (clash) throw new UsageError(`kaloko export ${sub} does not take --${clash}; that is kaloko export ${owner[clash]}.`)
976
+ if (sub === 'live') args.live = true
977
+ if (sub === 'design') args.design = typeof args.design === 'string' ? args.design : true
978
+ if (sub === 'docs' && typeof args.format !== 'string') args.format = 'md'
979
+ if (sub === 'steps' && typeof args.scenario !== 'string') throw new UsageError('kaloko export steps needs --scenario <id>: the flow whose steps to export.')
980
+ return sub
981
+ }
982
+
939
983
  async function affectedSelection (args, { config, root, scenario, scenarioFile }) {
940
984
  let client = null
941
985
  try { if (!args.offline) client = service(config) } catch {}
@@ -1062,13 +1106,18 @@ const commands = {
1062
1106
  async validate (args) {
1063
1107
  if (!args._.length) throw new UsageError('Usage: kaloko validate <scenario.yml>...')
1064
1108
  let failed = 0
1109
+ // --json: one entry per file with its problems, warnings and coverage; the lines below go to stderr
1110
+ const results = []
1065
1111
  for (const file of args._) {
1066
1112
  let scenario
1067
- try { scenario = parseScenario(await readFile(file, 'utf8'), file) } catch (e) { if (!(e instanceof UserError)) throw e; failed++; say(`✕ ${e.message}`); continue }
1113
+ try { scenario = parseScenario(await readFile(file, 'utf8'), file) } catch (e) { if (!(e instanceof UserError)) throw e; failed++; say(`✕ ${e.message}`); results.push({ file, ok: false, problems: [e.message], warnings: [] }); continue }
1068
1114
  const problems = validateScenario(scenario)
1069
1115
  if (!problems.length) expandPacks(scenario)
1070
1116
  if (problems.length) { failed++; say(`✕ ${file}\n - ${problems.join('\n - ')}`) } else say(`✓ ${file}`)
1071
- if (!problems.length) for (const w of scenarioWarnings(scenario)) say(` ! ${w}`)
1117
+ const warnings = problems.length ? [] : scenarioWarnings(scenario)
1118
+ for (const w of warnings) say(` ! ${w}`)
1119
+ const entry = { file, id: scenario.id || null, ok: !problems.length, problems, warnings }
1120
+ results.push(entry)
1072
1121
  if (args.coverage) {
1073
1122
  // the plan is scenario.source (relative to the working directory or to the scenario file), or --source
1074
1123
  const candidates = typeof args.source === 'string' ? [resolve(args.source)] : scenario.source ? [resolve(scenario.source), resolve(file, '..', scenario.source), resolve(file, '..', '..', scenario.source)] : []
@@ -1077,16 +1126,21 @@ const commands = {
1077
1126
  if (!plan) { say(` · no acceptance plan found (${scenario.source ? `source: ${scenario.source}` : 'scenario has no source'}); pass --source <ACCEPTANCE.md>`); continue }
1078
1127
  const cov = coverage(scenario, plan)
1079
1128
  say(formatCoverage(file, cov))
1129
+ entry.coverage = { missing: cov.missing }
1080
1130
  if (cov.missing.length) process.exitCode = 1
1081
1131
  }
1082
1132
  }
1083
1133
  if (failed) process.exitCode = 1
1084
- await warnRetired(args)
1134
+ const retired = await warnRetired(args)
1135
+ if (args.json) say(JSON.stringify({ ok: !failed && process.exitCode !== 1, files: results, retired }, null, 2))
1085
1136
  },
1086
1137
 
1087
1138
  async scenarios (args) {
1088
1139
  const { config, root } = await context(args)
1089
- for (const s of await catalogFor(config, root)) say(`${s.id.padEnd(36)} ${s.kind.padEnd(5)} ${String(s.runs.length).padStart(3)} runs ${s.file}`)
1140
+ const all = await catalogFor(config, root)
1141
+ // --json: id, title, kind, file and the local runs (newest first) of every scenario the config matches
1142
+ if (args.json) { say(JSON.stringify(all, null, 2)); return }
1143
+ for (const s of all) say(`${s.id.padEnd(36)} ${s.kind.padEnd(5)} ${String(s.runs.length).padStart(3)} runs ${s.file}`)
1090
1144
  },
1091
1145
 
1092
1146
  async start (args) {
@@ -1153,6 +1207,7 @@ const commands = {
1153
1207
  say(await stopBrowser({ root, config, env }) ? `Browser stopped${env ? ` (${env.name})` : ' (all environments)'}.` : 'Browser was not running.')
1154
1208
  } else {
1155
1209
  const states = await allStates({ root, config })
1210
+ if (args.json) { say(JSON.stringify(states.map((s) => ({ environment: s.environment, cdp: s.cdp, pid: s.pid, readonly: !!s.readonly })), null, 2)); return }
1156
1211
  say(states.length ? states.map((s) => `${s.environment}: ${s.cdp} (pid ${s.pid})${s.readonly ? ', readonly' : ''}`).join('\n') : 'No shared browser is running.')
1157
1212
  }
1158
1213
  },
@@ -1247,7 +1302,7 @@ const commands = {
1247
1302
  let since
1248
1303
  if (args['any-time']) since = null
1249
1304
  else if (typeof args.since === 'string') { const t = Date.parse(args.since); if (Number.isNaN(t)) throw new UserError(`--since ${args.since} is not a date (ISO 8601, e.g. 2026-09-27T10:00).`); since = new Date(t).toISOString() }
1250
- const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject, timeoutMs: (Number(args.timeout) || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
1305
+ const results = await captureMail({ config, root, run, env, scenario, node, locale, to, subject, timeoutMs: (parseDuration(args.timeout, 's', 'timeout') || 60) * 1000, viewports: list(args.viewports), message, since, id: typeof args.id === 'string' ? args.id : null })
1251
1306
  say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
1252
1307
  },
1253
1308
 
@@ -1504,10 +1559,13 @@ const commands = {
1504
1559
  out.push({ scenario: scenario.id, file: relative(root, file), base: r.base, changed: r.files, runs: r.runs, ...r.sel, walk: selectedIds(scenario, r.sel) })
1505
1560
  if (!args.json) say(formatAffected(r.sel, { scenario, base: r.base, changed: r.files.length }) + '\n')
1506
1561
  }
1562
+ const failIfNone = !!(args['fail-if-none'] || args.check)
1507
1563
  if (args.json) say(JSON.stringify(targets.length === 1 && typeof args.scenario === 'string' ? out[0] : out, null, 2))
1508
- else if (!args.check) say(`Walk them with: kaloko walk --affected${args.base ? ` --base ${args.base}` : ''} (after kaloko start)`)
1509
- // --check: exit code 1 when nothing is affected (CI skips the scenario)
1510
- if (args.check && !out.some((o) => o.walk.length && !o.none)) process.exitCode = 1
1564
+ else if (!failIfNone) say(`Walk them with: kaloko walk --affected${args.base ? ` --base ${args.base}` : ''} (after kaloko start)`)
1565
+ // --check is the older name of --fail-if-none: it works the same and says so once
1566
+ if (args.check && !args['fail-if-none']) console.error('! kaloko affected --check is now --fail-if-none: it works the same; rename it in your CI when you next touch it.')
1567
+ // --fail-if-none: exit code 1 when nothing is affected (CI skips the scenario)
1568
+ if (failIfNone && !out.some((o) => o.walk.length && !o.none)) process.exitCode = 1
1511
1569
  },
1512
1570
 
1513
1571
  /**
@@ -1583,7 +1641,7 @@ const commands = {
1583
1641
  const action = args._[0]
1584
1642
  if (action === 'preview-url') {
1585
1643
  const err = (m) => console.error(m)
1586
- const preview = await findPreview({ wait: Number(args.wait) || 0, say: err })
1644
+ const preview = await findPreview({ wait: parseDuration(args.wait, 's', 'wait') || 0, say: err })
1587
1645
  const pr = await pullRequestOf().catch(() => null)
1588
1646
  if (!preview) throw new UserError('No preview URL found. Set KALOKO_PREVIEW_URL, or let Vercel, Netlify or Cloudflare Pages report deployments to GitHub (the workflow needs deployments: read).')
1589
1647
  const lines = envLines({ preview, pr })
@@ -1593,6 +1651,8 @@ const commands = {
1593
1651
  }
1594
1652
  if (typeof args.dotenv === 'string') await writeEnvFile(resolve(args.dotenv), lines)
1595
1653
  err(`Preview ${preview.url} (from ${preview.source})${pr?.number ? `, pull request #${pr.number}${pr.base ? ` into ${pr.base}` : ''}` : ''}`)
1654
+ // --json: the URL, where it came from and the pull request, with the variables the env lines set
1655
+ if (args.json) { say(JSON.stringify({ url: preview.url, source: preview.source, pr: pr?.number || null, base: pr?.base || null, head: pr?.head || null, env: Object.fromEntries(lines.map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)])) }, null, 2)); return }
1596
1656
  console.log(preview.url)
1597
1657
  return
1598
1658
  }
@@ -1669,15 +1729,18 @@ const commands = {
1669
1729
  const { stat, mkdir, writeFile: write, chmod } = await import('node:fs/promises')
1670
1730
  if (action === 'list') {
1671
1731
  const rows = []
1732
+ const items = []
1672
1733
  for (const name of Object.keys(config.environments)) {
1673
1734
  let env
1674
1735
  try { env = environment(config, name) } catch { continue }
1675
1736
  for (const key of Object.keys(env.accounts || {})) {
1676
1737
  const file = accountFor(env, key, { root, config }).storage_state
1677
1738
  const st = await stat(file).catch(() => null)
1739
+ items.push({ environment: name, account: key, saved: !!st, saved_at: st ? new Date(st.mtimeMs).toISOString() : null, file: relative(process.cwd(), file) })
1678
1740
  rows.push(`${name.padEnd(12)} ${key.padEnd(16)} ${st ? `saved ${Math.round((Date.now() - st.mtimeMs) / 36e5)} h ago · ${relative(process.cwd(), file)}` : 'no saved session'}`)
1679
1741
  }
1680
1742
  }
1743
+ if (args.json) { say(JSON.stringify(items, null, 2)); return }
1681
1744
  say(rows.length ? rows.join('\n') : 'No accounts in kaloko.config.yml (environments.<env>.accounts).')
1682
1745
  return
1683
1746
  }
@@ -1780,6 +1843,11 @@ const commands = {
1780
1843
  say(`Semantic criteria not evaluated:\n - ${[...byReason].slice(0, 8).map(([why, where]) => `${where.length > 1 ? `${where.length} captures (${where.slice(0, 3).join(', ')}${where.length > 3 ? ', …' : ''})` : where[0]}: ${why}`).join('\n - ')}`)
1781
1844
  }
1782
1845
  if (report.outdated.length) say(`Criteria changed after capture of ${report.outdated.join(', ')}: not evaluated. Run "kaloko evaluate --recheck --steps ${report.outdated.join(',')}": it rechecks the stored captures against the new criteria, deterministic and semantic, without walking again.`)
1846
+ // --json: the counts and every verdict a person or agent has to act on (fail, needs_review)
1847
+ if (args.json) {
1848
+ const pick = (f) => ({ step: f.node, ...(f.item ? { item: f.item } : {}), locale: f.locale, viewport: f.viewport, criterion: f.id, verdict: f.verdict, why: f.why || null, fix: f.fix || null })
1849
+ say(JSON.stringify({ run: run.meta.id, evaluated: report.evaluated, semantic: { model: report.model, calls: report.semantic_calls, cached: report.semantic_cached, via: report.via.qawalk ? 'kaloko' : report.via['own-key'] ? 'own-key' : null }, vision: report.vision || null, triage: report.triage || null, findings: (report.findings || []).filter((f) => f.verdict === 'fail' || f.verdict === 'needs_review').map(pick), skipped: report.skipped, outdated: report.outdated }, null, 2))
1850
+ }
1783
1851
  },
1784
1852
 
1785
1853
  async build (args) {
@@ -1787,8 +1855,11 @@ const commands = {
1787
1855
  const manifest = await buildManifest({ run, scenario, config, root, catalog: await catalogFor(config, root) })
1788
1856
  const file = await writePreview({ run, manifest })
1789
1857
  const s = manifest.summary
1858
+ const why = attentionLines(manifest)
1859
+ // --json: the run id, its summary, the preview file and the AI verdicts to act on
1860
+ if (args.json) { say(JSON.stringify({ run: manifest.id, summary: s, preview: file, manifest: join(run.dir, 'run.json'), attention: why.map((l) => l.trim()) }, null, 2)); return file }
1790
1861
  say(`${manifest.id}: ${s.pass} pass · ${s.partial} partial · ${s.fail} fail · ${s.skip} not checked\nPreview: ${file}`)
1791
- const why = attentionLines(manifest); if (why.length) say(`AI verdicts to act on (score and why):\n${why.join('\n')}`)
1862
+ if (why.length) say(`AI verdicts to act on (score and why):\n${why.join('\n')}`)
1792
1863
  return file
1793
1864
  },
1794
1865
 
@@ -1801,7 +1872,7 @@ const commands = {
1801
1872
 
1802
1873
  async preview (args) {
1803
1874
  const file = await commands.build(args)
1804
- say(pathToFileURL(file).href)
1875
+ if (!args.json) say(pathToFileURL(file).href)
1805
1876
  },
1806
1877
 
1807
1878
  async share (args) {
@@ -1816,6 +1887,7 @@ const commands = {
1816
1887
  const meta = await readJson(join(run.dir, 'meta.json'))
1817
1888
  meta.shared = { url: v.url, at: new Date().toISOString(), version: v.version, run: v.id }
1818
1889
  await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
1890
+ if (args.json) say(JSON.stringify({ run: v.id, url: v.url, version: v.version ?? null, scenario: scenario.id }, null, 2))
1819
1891
  return
1820
1892
  }
1821
1893
  let manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
@@ -1846,7 +1918,10 @@ const commands = {
1846
1918
  if (findings.length && strict && !args['allow-pii']) throw new UserError(`Not shared: the run holds ${findings.length} unmasked personal data value(s):\n${formatFindings(findings)}\nMask them (capture.mask in the scenario, or pii: redact for automatic detection) and capture the steps again. Values that are not personal data, such as a company contact, go under pii.allow in kaloko.config.yml. To share anyway: kaloko share --allow-pii.`)
1847
1919
  if (findings.length && strict) say(`Sharing ${findings.length} unmasked personal data value(s), as --allow-pii says:\n${formatFindings(findings, 5)}`)
1848
1920
  else if (findings.length) say(`This run shows ${findings.length} value(s) that look like personal data:\n${formatFindings(findings, 5)}\nShared anyway, because the run does not hide personal data. To hide it, add pii: redact to the scenario or the environment; values that are not personal data go under pii.allow in kaloko.config.yml.`)
1849
- const days = Number(args.expires) || config.share?.expires_days || 30
1921
+ // --expires: days (30), or with a unit that makes whole days (whole days: 48h is 2)
1922
+ const asked = parseDuration(args.expires, 'd', 'expires')
1923
+ if (asked != null && (!Number.isInteger(asked) || asked < 1)) throw new UsageError(`--expires counts whole days: ${args.expires} is ${Math.round(asked * 100) / 100} days. Use 30, 30d or 72h.`)
1924
+ const days = asked || config.share?.expires_days || 30
1850
1925
  // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
1851
1926
  let entries = await hashFiles(run.dir, [...files])
1852
1927
  // a recording above the service's file limit stays local (with its poster and preview), the rest is shared
@@ -1870,7 +1945,7 @@ const commands = {
1870
1945
  } catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
1871
1946
  const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
1872
1947
  await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})
1873
- const up = await uploadRunFiles({ client, org: config.org, runId: manifest.id, dir: run.dir, entries, missing, say })
1948
+ const up = await uploadRunFiles({ client, org: config.org, runId: manifest.id, dir: run.dir, entries, missing, say, ...(args.json ? { write: (t) => process.stderr.write(t) } : {}) })
1874
1949
  if (up.mode === 'blobs') say(` ${up.uploaded} new file(s) sent (${fmtBytes(up.bytes)}), ${up.reused} unchanged file(s) reused`)
1875
1950
  // experimental HTML views (#42): their bundles travel as blobs and attach to the run as revisions
1876
1951
  const viewIds = new Set(manifest.nodes.flatMap((n) => Object.values(n.captures || {}).flatMap((vps) => Object.values(vps).map((c) => c.view?.revision).filter(Boolean))))
@@ -1896,10 +1971,14 @@ const commands = {
1896
1971
  if (review) say(review)
1897
1972
  const capped = expiryNote(days, created.expires_days)
1898
1973
  if (capped) say(`Note: ${capped}`)
1974
+ let prComment = null
1899
1975
  if (args.pr) {
1900
1976
  const r = await upsertPrComment({ manifest, url: fin.url, cwd: root })
1977
+ prComment = r.ok ? { action: r.action, url: r.url } : { skipped: r.reason }
1901
1978
  say(r.ok ? `PR comment ${r.action}: ${r.url}` : `PR comment skipped: ${r.reason}.`)
1902
1979
  }
1980
+ // --json: what a script needs next, the run id and its canvas URL (the lines above went to stderr)
1981
+ if (args.json) say(JSON.stringify({ run: manifest.id, url: fin.url, scenario: scenario.id, files: files.size, expires_days: created.expires_days, review: fin.review || null, ...(args.pr ? { pr_comment: prComment } : {}) }, null, 2))
1903
1982
  },
1904
1983
 
1905
1984
  async sync (args) {
@@ -2111,10 +2190,22 @@ const commands = {
2111
2190
  const client = service(config)
2112
2191
  const all = !!args.all || typeof args.scenario === 'string'
2113
2192
  const envName = envOption(args)
2114
- const q = new URLSearchParams({ ...(all ? {} : { latest: '1' }), ...(typeof args.scenario === 'string' ? { scenario: args.scenario } : {}), ...(envName ? { environment: envName } : {}), limit: String(RUNS_CAP) })
2115
- const runs = await client.request('GET', `/orgs/${config.org}/runs?${q}`)
2116
- // the service answers at most RUNS_CAP: say so (on stderr, so --json stays one JSON array)
2117
- if (runs.length >= RUNS_CAP) console.error(`Showing the newest ${RUNS_CAP}; there may be more. Narrow the list with --scenario <id> or --env <name>.`)
2193
+ // --limit N: at most N runs; --all follows the service's cursors to the last run, otherwise the newest RUNS_CAP;
2194
+ // --cursor continues where an earlier list stopped (the cursor it printed)
2195
+ if (args.limit !== undefined && !(Number(args.limit) >= 1)) throw new UsageError('--limit takes a number of runs, e.g. --limit 50.')
2196
+ const want = args.limit !== undefined ? Math.floor(Number(args.limit)) : args.all ? Infinity : RUNS_CAP
2197
+ const runs = []
2198
+ let cursor = typeof args.cursor === 'string' ? args.cursor : null
2199
+ let next = null
2200
+ do {
2201
+ const page = Math.min(1000, want - runs.length)
2202
+ const q = new URLSearchParams({ ...(all ? {} : { latest: '1' }), ...(typeof args.scenario === 'string' ? { scenario: args.scenario } : {}), ...(envName ? { environment: envName } : {}), limit: String(page), ...(cursor ? { cursor } : {}) })
2203
+ const got = await client.page(`/orgs/${config.org}/runs?${q}`)
2204
+ runs.push(...got.data)
2205
+ next = cursor = got.next
2206
+ } while (next && runs.length < want)
2207
+ // more runs than were listed: say how to go on (on stderr, so --json stays one JSON array)
2208
+ if (next) console.error(`Showing ${runs.length}; there are more. kaloko runs${args.all ? ' --all' : ''}${typeof args.scenario === 'string' ? ` --scenario ${args.scenario}` : ''}${envName ? ` --env ${envName}` : ''} --cursor ${next} lists the next ones${all && !args.all ? ', --all every one' : ''}.`)
2118
2209
  if (args.json) { say(JSON.stringify(runs, null, 2)); return }
2119
2210
  if (!runs.length) { say('No shared runs match.'); return }
2120
2211
  for (const r of runs) {
@@ -2140,8 +2231,8 @@ const commands = {
2140
2231
 
2141
2232
  async prune (args) {
2142
2233
  const { config } = await context(args)
2143
- const days = typeof args['older-than'] === 'string' ? Number(args['older-than'].replace(/d$/, '')) : null
2144
- if (days != null && !Number.isInteger(days)) throw new UsageError('--older-than takes days, e.g. 14d.')
2234
+ const days = parseDuration(args['older-than'], 'd', 'older-than')
2235
+ if (days != null && !Number.isInteger(days)) throw new UsageError(`--older-than counts whole days, e.g. 14, 14d or 48h; ${args['older-than']} is ${Math.round(days * 100) / 100} days.`)
2145
2236
  if (args.yes && args['dry-run']) throw new UsageError('--dry-run and --yes contradict each other: --dry-run (the default) lists, --yes moves runs to the trash.')
2146
2237
  const envName = envOption(args)
2147
2238
  const body = { dry_run: !args.yes, keep_latest: args.keep === undefined ? 3 : Number(args.keep), ...(typeof args.scenario === 'string' ? { scenario: args.scenario } : {}), ...(envName ? { environment: envName } : {}), ...(typeof args.project === 'string' ? { project: args.project } : {}), ...(days != null ? { older_than_days: days } : {}) }
@@ -2300,6 +2391,7 @@ const commands = {
2300
2391
  },
2301
2392
 
2302
2393
  async export (args) {
2394
+ exportSubcommand(args)
2303
2395
  const { config, root } = await context(args)
2304
2396
  const client = service(config)
2305
2397
  if (args.design) {
@@ -2370,7 +2462,7 @@ const commands = {
2370
2462
  const engines = []
2371
2463
  const scenarios = []
2372
2464
  for (const f of await scenarioFiles(config, root).catch(() => [])) { try { const s = yaml.load(await readFile(f, 'utf8')) || {}; engines.push(...(s.browsers || [])); scenarios.push({ file: relative(root, f), product: s.product || null, module: s.module || null }) } catch {} }
2373
- const r = await doctor({ config, root, file, fix: !!args.fix, engines, scenarios })
2465
+ const r = await doctor({ config, root, file, fix: !!args.fix, engines, scenarios, legacy: legacyInUse() })
2374
2466
  say(args.json ? JSON.stringify(r, null, 2) : formatDoctor(r))
2375
2467
  if (!r.ok) process.exitCode = 1
2376
2468
  },
@@ -2480,7 +2572,8 @@ const commands = {
2480
2572
  async help (args = { _: [] }) {
2481
2573
  const topic = args._?.[0]
2482
2574
  if (!topic) { say(HELP); return }
2483
- const text = commands[topic] && topic !== 'help' ? helpFor(HELP, topic, COMMON_HELP) : null
2575
+ // kaloko help export run: the entries of that sub-command (all of the command's when it has none by that name)
2576
+ const text = commands[topic] && topic !== 'help' ? helpFor(HELP, topic, COMMON_HELP, args._?.[1]) : null
2484
2577
  if (text) { say(text); return }
2485
2578
  unknownCommand(topic)
2486
2579
  }
@@ -2492,27 +2585,69 @@ async function packageVersion () {
2492
2585
  }
2493
2586
 
2494
2587
  /** "There is no command": the nearest command when it looks like a typo, the full help, exit code 2. */
2495
- function unknownCommand (name) {
2588
+ function unknownCommand (name, json = false) {
2496
2589
  const guess = closest(name, Object.keys(commands))
2497
- console.error(`There is no command "${name}".${guess ? ` Did you mean kaloko ${guess}?` : ''} The commands are below; kaloko help <command> explains one.\n`)
2590
+ const message = `There is no command "${name}".${guess ? ` Did you mean kaloko ${guess}?` : ''}`
2591
+ if (json) { console.error(JSON.stringify(jsonError(new UsageError(message, { next: 'kaloko help lists the commands.' })))); process.exitCode = EXIT.usage; return }
2592
+ console.error(`${message} The commands are below; kaloko help <command> explains one.\n`)
2498
2593
  say(HELP)
2499
- process.exitCode = 2
2594
+ process.exitCode = EXIT.usage
2595
+ }
2596
+
2597
+ /** A line of output that is a JSON document (the --json answer), told apart from progress and notes. */
2598
+ const isJsonText = (text) => { const t = text.trim(); if (!/^[[{]/.test(t)) return false; try { JSON.parse(t); return true } catch { return false } }
2599
+
2600
+ /** The --json error: { error: { message, code, next } } (help/reference/cli.md → Machine output). */
2601
+ export function jsonError (error) {
2602
+ const usage = error instanceof UsageError
2603
+ return { error: { message: error.reason || error.message, code: typeof error.code === 'string' ? error.code : usage ? 'usage' : error instanceof UserError ? 'failed' : 'internal', next: error.next || (usage ? 'kaloko help <command> shows how to use it.' : null) } }
2500
2604
  }
2501
2605
 
2502
- export async function main (argv) {
2606
+ /** The exit code of an error: 2 usage, 3 the service cannot be reached, 4 no or a bad token, else 1. */
2607
+ export const exitCodeOf = (error) => error?.exitCode || (error instanceof UsageError ? EXIT.usage : EXIT.failed)
2608
+
2609
+ /**
2610
+ * Runs one command. `bin` is the name it was started by: the qawalk command from before the rename still works.
2611
+ * With --json, stdout carries only the JSON answer: every other line goes to stderr, and an error is one JSON line
2612
+ * on stderr ({ error: { message, code, next } }) with the same exit code as without --json.
2613
+ */
2614
+ export async function main (argv, { bin = 'kaloko' } = {}) {
2615
+ resetLegacy()
2616
+ if (bin === 'qawalk') noteLegacy('the qawalk command', 'kaloko')
2503
2617
  const args = parseArgs(argv)
2504
- if ((!args._.length && args.version) || args._[0] === 'version') { say(await packageVersion()); return }
2618
+ const json = !!args.json
2619
+ const stdout = console.log
2620
+ if (json) console.log = (...m) => (isJsonText(m.join(' ')) ? stdout : console.error)(...m)
2621
+ try {
2622
+ await runCommand(args, json)
2623
+ } finally {
2624
+ if (json) console.log = stdout
2625
+ const hint = legacyHint()
2626
+ if (hint) console.error(hint)
2627
+ }
2628
+ }
2629
+
2630
+ async function runCommand (args, json) {
2631
+ if ((!args._.length && args.version) || args._[0] === 'version') { say(json ? JSON.stringify({ version: await packageVersion() }) : await packageVersion()); return }
2505
2632
  const name = args._.shift() || 'help'
2506
2633
  const cmd = commands[name]
2507
- if (!cmd) { if (args.help) say(HELP); else unknownCommand(name); return }
2634
+ if (!cmd) { if (args.help) say(HELP); else unknownCommand(name, json); return }
2508
2635
  // kaloko <command> --help: the command's help instead of running it
2509
- if (args.help && name !== 'help') { say(helpFor(HELP, name, COMMON_HELP) || HELP); return }
2636
+ if (args.help && name !== 'help') { say(helpFor(HELP, name, COMMON_HELP, args._[0]) || HELP); return }
2510
2637
  try {
2511
2638
  for (const line of checkOptions(HELP, name, args)) console.error(line)
2512
2639
  await cmd(args)
2513
2640
  } catch (error) {
2641
+ const known = error instanceof UserError
2642
+ if (json) {
2643
+ // the reason in one JSON line on stderr, the documented exit code; a bug also prints its stack before it
2644
+ if (!known) console.error(error?.stack || String(error))
2645
+ console.error(JSON.stringify(jsonError(error)))
2646
+ process.exitCode = exitCodeOf(error)
2647
+ return
2648
+ }
2514
2649
  // a usage mistake: the command's help says what is wrong; no support line, exit code 2
2515
- if (error instanceof UsageError) { console.error(`${error.message}${/kaloko help /.test(error.message) ? '' : `\nkaloko help ${name} shows how to use it.`}`); process.exitCode = 2 } else if (error instanceof UserError) { console.error(`${error.message}\n${SUPPORT_LINE}`); process.exitCode = 1 } else {
2650
+ if (error instanceof UsageError) { console.error(`${error.message}${/kaloko help /.test(error.message) ? '' : `\nkaloko help ${name} shows how to use it.`}`); process.exitCode = EXIT.usage } else if (known) { console.error(`${error.message}\n${SUPPORT_LINE}`); process.exitCode = exitCodeOf(error) } else {
2516
2651
  console.error(`Kaloko stopped on an unexpected error (a bug on our side). Send the output below to ${SUPPORT_EMAIL} with the command you ran.`)
2517
2652
  throw error
2518
2653
  }