kaloko 0.4.10 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -113,6 +113,24 @@ jobs:
113
113
  Uploads go by content hash: captures that did not change since an earlier run
114
114
  are not sent again, and an interrupted `share` picks up where it stopped.
115
115
 
116
+ ## When a screenshot is not enough
117
+
118
+ ```bash
119
+ npx kaloko walk --record # each step on video, from the previous screen to its capture
120
+ npx kaloko walk --record failed # only the steps that fail (flaky ones, mostly)
121
+ npx kaloko walk --record flow # the whole walk per language, a chapter per step
122
+ npx kaloko video export --format mp4 # the run as one video, a title card and verdict before each step
123
+ ```
124
+
125
+ The canvas plays a step's video next to its screenshot, and the Technical tab
126
+ shows its trace: clicks, typing, requests with status and timing, console
127
+ messages. Click a row and the video jumps there. Web recordings need nothing
128
+ but Chrome (WebM with VP9); with ffmpeg installed you also get MP4 for Safari,
129
+ app recordings (Android `screenrecord`, iOS Simulator, macOS) and the
130
+ walkthrough export. `npx kaloko video tools` says what this machine has.
131
+ Recordings stay in the local preview on Free and Team; Business and Enterprise
132
+ share them with the run.
133
+
116
134
  ## Screenshots you already have
117
135
 
118
136
  ```bash
@@ -193,6 +211,7 @@ Nothing in Kaloko is stored in a format only Kaloko can read. Every export opens
193
211
  | Live files (handoff) | `npx kaloko export --live [--version n] --out design/` | the design folder as written |
194
212
  | Shared run | `npx kaloko export --run <id>` | zip with `run.json` (`qawalk.run.v1`), screenshots and HTML snapshots, live HTML of design steps, comments and verdicts, and `index.html`: the canvas, offline |
195
213
  | Documentation captures | `npx kaloko export --scenario <id>` | newest image and HTML of every step, `index.json` |
214
+ | Walkthrough video | `npx kaloko video export [--format mp4\|webm\|gif]` | one video of the run's recordings with title cards; step recordings are WebM (VP9) and MP4 (H.264), traces JSON (`qawalk.trace.v1`) |
196
215
 
197
216
  The schemas of these formats ship with the package (`schemas/`).
198
217
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "kaloko",
3
- "version": "0.4.10",
3
+ "version": "0.4.11",
4
4
  "description": "See what your AI agents built: every screen, language and device walked, assessed and approved on one canvas.",
5
5
  "keywords": [
6
6
  "qa",
@@ -81,6 +81,15 @@
81
81
  "concurrency": { "type": "integer", "minimum": 1, "maximum": 16, "description": "Locales walked at the same time, each in its own browser context (kaloko walk --concurrency). Default: by CPU in an own browser, one at a time on the shared browser." }
82
82
  }
83
83
  },
84
+ "capture": {
85
+ "type": "object",
86
+ "additionalProperties": false,
87
+ "description": "Defaults for every scenario of the project; a scenario's capture block wins.",
88
+ "properties": {
89
+ "record": { "$ref": "#/$defs/record" },
90
+ "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
91
+ }
92
+ },
84
93
  "share": {
85
94
  "type": "object",
86
95
  "additionalProperties": false,
@@ -92,6 +101,25 @@
92
101
  }
93
102
  },
94
103
  "$defs": {
104
+ "record": {
105
+ "description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
106
+ "oneOf": [
107
+ { "type": "boolean" },
108
+ { "enum": ["step", "flow", "failed", "off"] },
109
+ {
110
+ "type": "object",
111
+ "additionalProperties": false,
112
+ "properties": {
113
+ "mode": { "enum": ["step", "flow", "failed", "off"], "default": "step" },
114
+ "quality": { "enum": ["low", "medium", "high"], "default": "medium", "description": "low 8 fps up to 960 px, medium 12 fps up to 1280 px, high 24 fps up to 1920 px" },
115
+ "max_seconds": { "type": "integer", "minimum": 1, "maximum": 3600, "description": "Longest kept recording; a longer one keeps its end (default 60 for a step, 900 for a flow)" },
116
+ "codec": { "enum": ["auto", "vp9", "av1", "h264"], "default": "auto", "description": "auto: WebM VP9 (VP8 fallback) from the browser, plus MP4 H.264 when ffmpeg is installed; av1 needs ffmpeg" },
117
+ "mp4": { "oneOf": [{ "type": "boolean" }, { "const": "auto" }], "default": "auto", "description": "MP4 H.264 next to the WebM (Safari before iOS 17.4); needs ffmpeg" },
118
+ "cursor": { "type": "boolean", "default": true, "description": "Draw the pointer and clicks into web recordings" }
119
+ }
120
+ }
121
+ ]
122
+ },
95
123
  "environment": {
96
124
  "type": "object",
97
125
  "additionalProperties": false,
@@ -163,6 +163,25 @@
163
163
  }
164
164
  },
165
165
  "$defs": {
166
+ "record": {
167
+ "description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
168
+ "oneOf": [
169
+ { "type": "boolean" },
170
+ { "enum": ["step", "flow", "failed", "off"] },
171
+ {
172
+ "type": "object",
173
+ "additionalProperties": false,
174
+ "properties": {
175
+ "mode": { "enum": ["step", "flow", "failed", "off"], "default": "step" },
176
+ "quality": { "enum": ["low", "medium", "high"], "default": "medium", "description": "low 8 fps up to 960 px, medium 12 fps up to 1280 px, high 24 fps up to 1920 px" },
177
+ "max_seconds": { "type": "integer", "minimum": 1, "maximum": 3600, "description": "Longest kept recording; a longer one keeps its end (default 60 for a step, 900 for a flow)" },
178
+ "codec": { "enum": ["auto", "vp9", "av1", "h264"], "default": "auto", "description": "auto: WebM VP9 (VP8 fallback) from the browser, plus MP4 H.264 when ffmpeg is installed; av1 needs ffmpeg" },
179
+ "mp4": { "oneOf": [{ "type": "boolean" }, { "const": "auto" }], "default": "auto", "description": "MP4 H.264 next to the WebM (Safari before iOS 17.4); needs ffmpeg" },
180
+ "cursor": { "type": "boolean", "default": true, "description": "Draw the pointer and clicks into web recordings" }
181
+ }
182
+ }
183
+ ]
184
+ },
166
185
  "slug": {
167
186
  "type": "string",
168
187
  "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
@@ -269,6 +288,7 @@
269
288
  "capture": {
270
289
  "$ref": "#/$defs/capture"
271
290
  },
291
+ "record": { "$ref": "#/$defs/record", "description": "Record this step even when the walk does not (true), or never (false)." },
272
292
  "ignore": {
273
293
  "type": "array",
274
294
  "description": "Regions left out of pixel comparisons (clocks, ads, carousels): a selector, or a rectangle in page pixels of one viewport. kaloko stability suggests them.",
@@ -710,7 +730,9 @@
710
730
  "full_page": {
711
731
  "type": "boolean",
712
732
  "description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
713
- }
733
+ },
734
+ "record": { "$ref": "#/$defs/record" },
735
+ "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
714
736
  }
715
737
  }
716
738
  }
@@ -13,7 +13,8 @@ description: >
13
13
  design environment (live HTML steps, drafts, versions, the design
14
14
  reference, checking implementation against it), and when implementing a
15
15
  screen that has an approved design (read the step spec, build with its
16
- tokens, check drift).
16
+ tokens, check drift), and when a step should be recorded on video or its
17
+ trace read (actions, requests, console).
17
18
  ---
18
19
 
19
20
  # Kaloko
@@ -50,7 +51,9 @@ create test data there (users, roles, carts), capture screens and e-mails,
50
51
  evaluate, build the preview, and fix scenario mechanics (selectors, scripts)
51
52
  that do not change what is being accepted, including applying a selector
52
53
  proposal with `kaloko walk --heal` when it names the same element. Walk only
53
- the steps a change reaches (`--affected`) and import screenshots into a run. On a design environment: change
54
+ the steps a change reaches (`--affected`) and import screenshots into a run. Record steps locally
55
+ (`walk --record`) when a screenshot cannot show what a person has to judge, read traces, and
56
+ export a walkthrough video into the run folder. On a design environment: change
54
57
  the design folder, walk, and sync the draft (including taking others' steps
55
58
  with `--theirs`). Move to the trash runs you uploaded in this session that
56
59
  were superseded before anyone looked at them (debug rounds: no comments, no
@@ -62,7 +65,8 @@ Changing acceptance criteria or scope, any production run, writing or
62
65
  replacing a CI workflow (`kaloko ci github|gitlab`), uploading with
63
66
  `kaloko share` (on a design environment: publishing a version), setting a
64
67
  design reference or baseline, merging a design branch, keeping your version of
65
- a step in a sync conflict (`--mine`), turning on `html_view`, a real clean-up
68
+ a step in a sync conflict (`--mine`), turning on `html_view`, turning on recordings for
69
+ a whole project (`capture.record` in the config: storage), a real clean-up
66
70
  (`kaloko prune --yes`), trashing someone else's run, archiving a scenario, and
67
71
  anything the environment profile blocks. Never trash an accepted run, a
68
72
  baseline, a run with review comments, or a run linked from an open PR or issue
@@ -93,6 +97,9 @@ they can do; never write secrets into the config, a scenario or a script.
93
97
  (`--heal`): [reference/execution.md](reference/execution.md)
94
98
  - Screenshots, walkthroughs, Playwright reports and traces that an agent or a
95
99
  tool already produced, laid out as a run: [reference/import.md](reference/import.md)
100
+ - Video of a step or a whole flow, traces (actions, requests, console) and the
101
+ walkthrough video; when recording is worth its storage:
102
+ [reference/recording.md](reference/recording.md)
96
103
  - CI: a workflow that walks each pull request's preview deployment and
97
104
  comments on it (`kaloko ci github|gitlab`): [reference/ci.md](reference/ci.md)
98
105
  - Evaluation, JEV thresholds and reading results: [reference/evaluation.md](reference/evaluation.md)
@@ -28,6 +28,7 @@ file.
28
28
  | `list_notifications` | the token owner's inbox (`unread_only`, `limit`) |
29
29
  | `list_projects` | projects of the organization with counts; the slug is the `project` filter of `list_runs` and the value for `project:` in a scenario |
30
30
  | `get_criteria_verdicts` | reviewer verdicts on manual or overridden criteria of a run |
31
+ | `get_step_recording` | a step's recording and trace (`run_id`, `step`, `locale`, `viewport`, `trace: true` for the full trace): video URLs, duration, its chapter of a flow recording, failed attempts, the trace summary (failed requests, console errors, slowest request); see [recording.md](recording.md). `get_run_summary` lists trace summaries per step too |
31
32
  | `get_review_queue` | what a person still has to decide in a run (`run_id`, or `scenario` / nothing for the newest run): steps in review order with reasons, decisions, approvals carried over unchanged steps with their origin, counts and `review_url` to give the person |
32
33
  | `decide_step` | records your (the agent's) decision on a step: `approve`, or `return` with a `note`; marked as an agent's and not counted as a person's. Never use it to approve for a person |
33
34
  | `revoke_carried_approval` | revokes an approval carried over to a step that looks unchanged but should be looked at again (`run_id`, `step`) |
@@ -0,0 +1,95 @@
1
+ # Recordings, traces and the walkthrough video
2
+
3
+ A screenshot shows where a step ended. A recording shows how it got there:
4
+ the clicks, the spinner that never stopped, the menu that opened twice. A trace
5
+ lists what happened meanwhile: actions, network requests with status, size and
6
+ timing, and console messages.
7
+
8
+ ## When to record
9
+
10
+ Record when a screenshot cannot show the problem or the behaviour a reviewer
11
+ has to judge:
12
+
13
+ - a flaky step (passes, then fails on the same commit): `--record failed` keeps
14
+ only the attempts that fail;
15
+ - animations, transitions, loading states, drag and drop, menus and dialogs
16
+ that open and close;
17
+ - a bug report or a PR where the person needs to see the interaction once;
18
+ - a demo for a client or a release note (`kaloko video export`).
19
+
20
+ Do not record every walk by default. Recordings are much larger than
21
+ screenshots, count in the organization's storage and expire after 30 days on
22
+ Business. Traces are small and are kept for every walked web step anyway.
23
+
24
+ ## Commands
25
+
26
+ ```bash
27
+ kaloko walk --record # each step: from the previous screen to its capture
28
+ kaloko walk --record flow # the whole walk per locale, a chapter per step
29
+ kaloko walk --record failed # only steps whose script fails or whose checks fail
30
+ kaloko walk --record --record-quality low --steps checkout
31
+ kaloko walk --no-trace # no trace either
32
+ kaloko video list # what was recorded and traced, with sizes
33
+ kaloko video trace --step checkout # the step's trace as a timeline in the terminal
34
+ kaloko video export --format mp4 # the run's recordings with a title card per step
35
+ kaloko video tools # ffmpeg, adb, simctl on this machine
36
+ ```
37
+
38
+ In a scenario, or in `kaloko.config.yml` for the whole project:
39
+
40
+ ```yaml
41
+ capture:
42
+ record: { mode: failed, quality: low, max_seconds: 45 } # or true, step, flow, failed, false
43
+ trace: true
44
+ nodes:
45
+ - id: checkout
46
+ record: true # this step always; record: false never
47
+ ```
48
+
49
+ Quality presets: `low` 8 fps up to 960 px, `medium` (default) 12 fps up to
50
+ 1280 px, `high` 24 fps up to 1920 px. A step keeps its last 60 seconds and a
51
+ flow its last 15 minutes unless `max_seconds` says otherwise.
52
+
53
+ ## What you get
54
+
55
+ Per recorded step under `steps/<step>/<locale>/<viewport>.`: `rec.webm` (VP9
56
+ from Chrome, no extra tools), `rec.mp4` (H.264, when ffmpeg is installed; Safari
57
+ before iOS 17.4 needs it), `rec.poster.webp`, `rec.webp` (a few-second animated
58
+ preview; `rec.gif` where the browser cannot encode WebP) and `trace.json`. The
59
+ pointer and clicks are drawn into web recordings. A flow recording is
60
+ `attachments/flow/<locale>.rec.*`; the walkthrough is
61
+ `attachments/walkthrough.rec.<format>`.
62
+
63
+ Web and Electron record through Chrome's screencast; walks in Firefox or WebKit
64
+ are neither recorded nor traced. Apps: Android with `adb shell screenrecord`
65
+ (segments, touches shown), the iOS Simulator with `simctl io recordVideo`,
66
+ macOS desktop apps with `screencapture` (Screen Recording permission). App
67
+ recordings need ffmpeg for the poster and preview; `kaloko doctor` lists what is
68
+ missing with the install command for the OS.
69
+
70
+ ## On the canvas
71
+
72
+ The step detail gets Video next to Screenshot: the poster first, nothing loads
73
+ until played, markers for the start, the capture, actions and failed requests.
74
+ Technical shows the trace; a click on a row moves the video there. Cards play
75
+ the preview on hover. Replay plays the flow recording with its chapters. The
76
+ walkthrough video is in the top bar.
77
+
78
+ ## Plans and sharing
79
+
80
+ Recording is local and works on every plan. `kaloko share` uploads recordings
81
+ on Business and Enterprise; on Free and Team it says they stay in the local
82
+ preview and shares screenshots and traces. A recording over the service's file
83
+ limit (25 MB) stays local too: lower the quality or `max_seconds`. Recordings
84
+ expire 30 days after the run is shared (Enterprise chooses in Settings →
85
+ Storage); the screenshots stay as long as the run. Uploading is still "ask
86
+ first", and so is turning recordings on for a whole project.
87
+
88
+ ## Reading them as an agent
89
+
90
+ `kaloko video trace --step <id>` locally, or MCP `get_step_recording`
91
+ (`run_id`, `step`; `trace: true` adds the full trace) on a shared run. It
92
+ answers the video URLs, duration, the step's chapter of a flow recording,
93
+ failed attempts and the trace summary: failed requests with status, console
94
+ errors, the slowest request. Fetch the URLs with the same token. Start there
95
+ when a step failed and the evidence in the criteria is not enough.
package/src/cli.mjs CHANGED
@@ -51,6 +51,9 @@ import { exportDesign } from './lib/design-export.mjs'
51
51
  import { driverFor, isApp, pngSize, ocr as ocrImage } from './lib/apps/drivers.mjs'
52
52
  import { captureApp } from './lib/apps/capture.mjs'
53
53
  import { readTree, flatten } from './lib/apps/tree.mjs'
54
+ import { recordSettings, recordingSession, recordingFiles, traceFiles, stripRecordings, withoutRecordings } from './lib/recording.mjs'
55
+ import { nativeSession } from './lib/apps/recording-native.mjs'
56
+ import { videoCmd } from './lib/video-cli.mjs'
54
57
 
55
58
 
56
59
  const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
@@ -130,6 +133,11 @@ Run
130
133
  steps' captures from the previous local run
131
134
  [--heal | --no-heal] A selector the page lost gets a proposal from the page (test id, role and name,
132
135
  text) with a patch; --heal applies it to the step script once the step passes
136
+ [--record [step|flow|failed]] [--record-quality low|medium|high] [--no-trace]
137
+ Record each step from the previous screen to its capture (step), the whole walk
138
+ per locale with a chapter per step (flow), or only steps that fail (failed);
139
+ also capture.record in the config or scenario, record: true on a step.
140
+ A trace (actions, requests, console) is kept per step unless --no-trace
133
141
  affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--check] [--offline]
134
142
  Which steps changed files reach, and why: framework route files against step paths
135
143
  and URLs captured earlier, imports, global files, i18n messages, step scripts and
@@ -147,6 +155,11 @@ Results
147
155
  the semantic questions again on the stored captures (no walk);
148
156
  console, overflow and performance keep their live result
149
157
  build Write run.json and the local preview (index.html)
158
+ video export [--format mp4|webm|gif] [--locale l] [--steps a,b] [--lang en|cs] [--out file]
159
+ The run's recordings in flow order with a title card per step (title, verdict);
160
+ shared with the run on Business and up
161
+ video list | trace --step <id> | tools
162
+ Recordings and traces of the run; a step's trace as a timeline; ffmpeg, adb, simctl
150
163
  preview Build and print the preview path
151
164
  share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
152
165
  hash, so unchanged captures are not sent again and a broken-off share resumes;
@@ -505,6 +518,14 @@ function linkFor (env, node, locale) {
505
518
  if (/^[a-z][a-z0-9+.-]*:/i.test(p)) return p
506
519
  return env.base_url ? urlFor(env, { path: p }, locale) : null
507
520
  }
521
+ /** A headless Chrome of its own for encoding recordings, started on first use (one per walk). */
522
+ function lazyEncoder (config) {
523
+ let started = null
524
+ return {
525
+ page: () => (started ||= launchEphemeral(config).then(async (b) => ({ b, p: await b.newPage() })).catch(() => ({ b: null, p: null }))).then((x) => x.p),
526
+ close: async () => { if (started) await (await started).b?.close().catch(() => {}) }
527
+ }
528
+ }
508
529
  const appViewport = (run) => Object.keys(run.meta.scope.viewports || {})[0] || null
509
530
 
510
531
  /**
@@ -522,6 +543,8 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
522
543
  const encoder = enc ? await enc.newPage().catch(() => null) : null
523
544
  const streak = failureStreak(args['keep-going'] ? 0 : 3)
524
545
  const maxChars = config.evaluators?.jev?.max_state_chars
546
+ // recordings and traces (batch D): Electron windows use the screencast recorder, devices record themselves
547
+ const recSettings = recordSettings({ args, config, scenario })
525
548
  try {
526
549
  for (const locale of locales) {
527
550
  const driver = driverFor(platform, env, { root, scenarioDir, locale })
@@ -530,6 +553,9 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
530
553
  if (driver.web) observeConsole(driver.page)
531
554
  say(`${platform} ${locale}: ${driver.web ? 'Electron window' : 'app'} ready`)
532
555
  } catch (error) { failed.push(`${platform} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${platform} ${locale}: ${error.message.split('\n')[0]}`); await driver.close().catch(() => {}); if (error instanceof UserError) break; continue }
556
+ const rec = driver.web
557
+ ? await recordingSession({ page: driver.page, run, scenario, locale, settings: recSettings, viewports: run.meta.scope.viewports, encoder: async () => encoder, say, label: `${platform} ${locale}` }).catch(() => null)
558
+ : await nativeSession({ platform, env, run, scenario, locale, settings: recSettings, driver, encoder: async () => encoder, say }).catch(() => null)
533
559
  try {
534
560
  for (const node of scenario.nodes) {
535
561
  if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
@@ -542,30 +568,42 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
542
568
  const link = linkFor(env, node, locale)
543
569
  if (!scripted && !(link && ['playwright', 'script'].includes(node.execution?.strategy))) { pending.push(`${node.id} (${locale}): ${node.execution?.strategy || 'agent'} — ${(node.execution?.instructions || '').split('\n')[0]}`); continue }
544
570
  if (streak.stopped(locale)) { skipped.push(`${node.id} (${locale})`); continue }
571
+ const records = []
545
572
  try {
546
573
  let captured = false
574
+ await rec?.beforeStep(node)
547
575
  const capture = async (o = {}) => {
548
576
  captured = true
577
+ await rec?.beforeCapture(node)
549
578
  if (driver.web) {
550
579
  const [w, h] = await driver.page.evaluate(() => [innerWidth, innerHeight])
551
- return captureStep({ page: driver.page, node, locale, options: captureOptions(scenario, node), viewports: { [o.viewport || appViewport(run) || 'window']: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: 'Electron' } })
580
+ const r = await captureStep({ page: driver.page, node, locale, options: captureOptions(scenario, node), viewports: { [o.viewport || appViewport(run) || 'window']: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: 'Electron' } })
581
+ records.push(...r)
582
+ return r
552
583
  }
553
- return captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: o.viewport || appViewport(run), run, platform, encoder, maxChars, extra: { url: link } })
584
+ const r = await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: o.viewport || appViewport(run), run, platform, encoder, maxChars, extra: { url: link } })
585
+ records.push(r)
586
+ return r
554
587
  }
555
588
  if (scripted) {
556
589
  const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
557
590
  await mod.default({ app: driver, page: driver.page || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
558
591
  } else await driver.link(link)
559
592
  if (!captured) await capture()
593
+ await rec?.afterStep(node, driver.web ? { records } : { record: records[0] })
560
594
  say(`✓ ${node.id} ${locale}`)
561
595
  streak.ok(locale)
562
596
  } catch (error) {
597
+ await rec?.afterStep(node, driver.web ? { error, records } : { error, record: records[0] }).catch(() => {})
563
598
  failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
564
599
  say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
565
600
  if (streak.record(locale, error.message)) say(` stopping ${locale}: the last 3 steps failed with the same error`)
566
601
  }
567
602
  }
568
- } finally { await driver.close().catch(() => {}) }
603
+ } finally {
604
+ for (const n of (await rec?.finish().catch((e) => [`recording: ${e.message}`])) || []) say(` ${n}`)
605
+ await driver.close().catch(() => {})
606
+ }
569
607
  }
570
608
  } finally { await enc?.close().catch(() => {}) }
571
609
  if (pending.length) say(`\nSteps for an agent or a person (drive the app, then kaloko capture --step <id> --image shot.png [--tree dump.xml|json]):\n - ${[...new Set(pending)].join('\n - ')}`)
@@ -861,6 +899,11 @@ const commands = {
861
899
  say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
862
900
  },
863
901
 
902
+ async video (args) {
903
+ const { config, root } = await context(args)
904
+ await videoCmd({ config, root, args, say })
905
+ },
906
+
864
907
  async walk (args) {
865
908
  const { config, root, run, scenario, env } = await runContext(args)
866
909
  if (args.affected) {
@@ -905,6 +948,9 @@ const commands = {
905
948
  // semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
906
949
  const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root })
907
950
  const walked = await readCaptures(run.dir)
951
+ // recordings and traces (batch D, #12): opt-in recordings, traces unless turned off; encoded in an own headless Chrome
952
+ const recSettings = recordSettings({ args, config, scenario })
953
+ const encoder = lazyEncoder(config)
908
954
  const sessions = scenario.sessions || null
909
955
  // named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
910
956
  const sessionContexts = new Map()
@@ -939,6 +985,12 @@ const commands = {
939
985
  const page = await lctx.newPage()
940
986
  const shared = conn && !parallel // the shared browser's own context logs its console to a file
941
987
  if (!shared) observeConsole(page)
988
+ // one recorder per page (the walk's tab and each named session's), started on its first step
989
+ const recorders = new Map()
990
+ const recorderFor = async (p) => {
991
+ if (!recorders.has(p)) recorders.set(p, recordingSession({ page: p, run, scenario, locale, settings: recSettings, viewports: run.meta.scope.viewports, encoder: encoder.page, say, label: locale }).catch(() => null))
992
+ return recorders.get(p)
993
+ }
942
994
  try {
943
995
  // every locale starts as a new visitor on its first walk in the run (an own context is new already)
944
996
  if (shared && !walked.some((c) => c.locale === locale)) { await resetSession(lctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale})`) }
@@ -978,11 +1030,17 @@ const commands = {
978
1030
  const scriptFile = resolve(scenarioDir, node.execution.script)
979
1031
  // one invocation of the step script; the automatic capture runs unless the script captured the step itself:
980
1032
  // a crop is an attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
1033
+ const rec = await recorderFor(sPage)
981
1034
  const runStep = async (mod) => {
982
1035
  let captured = false
983
- const capture = (opts = {}) => { captured = true; return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); return r }) }
984
- await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
985
- if (!captured) await capture()
1036
+ rec?.beforeStep(node)
1037
+ const records = []
1038
+ const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
1039
+ try {
1040
+ await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
1041
+ if (!captured) await capture()
1042
+ } catch (error) { await rec?.afterStep(node, { error }); throw error }
1043
+ await rec?.afterStep(node, { records })
986
1044
  }
987
1045
  try {
988
1046
  await runStep(await import(pathToFileURL(scriptFile).href))
@@ -1002,7 +1060,10 @@ const commands = {
1002
1060
  if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
1003
1061
  }
1004
1062
  }
1005
- } finally { await page.close().catch(() => {}) }
1063
+ } finally {
1064
+ for (const r of recorders.values()) { const notes = await (await r)?.finish().catch((e) => [`recording: ${e.message}`]); for (const n of notes || []) say(` ${n}`) }
1065
+ await page.close().catch(() => {})
1066
+ }
1006
1067
  return { pending, failed, skipped }
1007
1068
  }
1008
1069
  let results
@@ -1014,6 +1075,7 @@ const commands = {
1014
1075
  for (const c of ownContexts) await c.close().catch(() => {})
1015
1076
  if (conn) await conn.browser.close() // over CDP this only disconnects; the shared browser keeps running
1016
1077
  if (ephemeral) await ephemeral.close()
1078
+ await encoder.close()
1017
1079
  await releaseLock?.()
1018
1080
  }
1019
1081
  const pending = results.flatMap((r) => r.pending); const failed = results.flatMap((r) => r.failed); const skipped = results.flatMap((r) => r.skipped)
@@ -1107,7 +1169,7 @@ const commands = {
1107
1169
  const kept = await keepVideos(run, source.videos)
1108
1170
  say(`\nImported ${captured} screen(s)${attached ? `, ${attached} more as attachments` : ''} into ${run.meta.id}.${encoder ? '' : ' (No Chrome: images kept as they are, without thumbnails.)'}`)
1109
1171
  if (unmatched.length) say(`Not matched to a step (${unmatched.length}): name the step in steps.json ("step": "<id>"), or pass --map "${unmatched[0]}=<step id>". Steps: ${scenario.nodes.filter((n) => n.kind !== 'decision').map((n) => n.id).join(', ')}`)
1110
- if (kept.length) say(`Recordings kept in the run folder, not uploaded (the canvas does not play them yet): ${kept.join(', ')}`)
1172
+ if (kept.length) say(`Recordings kept in the run folder, not attached to a step and not uploaded: ${kept.join(', ')}. kaloko walk --record records steps for the canvas.`)
1111
1173
  say('Next: kaloko evaluate, then kaloko preview (local canvas) or kaloko share. Imported screens have no page to check: their deterministic criteria wait for a walk, semantic ones for a person.')
1112
1174
  },
1113
1175
 
@@ -1311,9 +1373,16 @@ const commands = {
1311
1373
  await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
1312
1374
  return
1313
1375
  }
1314
- const manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
1376
+ let manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
1315
1377
  await writePreview({ run, manifest })
1316
- const files = new Set()
1378
+ // the plan and storage before anything is sent; recordings travel on plans with the recordings feature only
1379
+ const u = await client.request('GET', `/orgs/${config.org}/usage`).catch(() => null)
1380
+ const recs = recordingFiles(manifest)
1381
+ if (recs.length && u?.limits?.features && !u.limits.features.recordings) {
1382
+ manifest = stripRecordings(manifest)
1383
+ say(`Recordings stay in the local preview (${recs.length} file(s)): sharing them is part of the Business plan and up, and this organization is on ${u.limits.plan_name}. Screenshots and traces are shared as usual.`)
1384
+ }
1385
+ const files = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
1317
1386
  for (const n of manifest.nodes) {
1318
1387
  for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree, c.spec].filter(Boolean).forEach((p) => files.add(p))
1319
1388
  for (const a of n.attachments || []) files.add(a.file)
@@ -1326,16 +1395,25 @@ const commands = {
1326
1395
  }
1327
1396
  const days = Number(args.expires) || config.share?.expires_days || 30
1328
1397
  // by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
1329
- const entries = await hashFiles(run.dir, [...files])
1398
+ let entries = await hashFiles(run.dir, [...files])
1399
+ // a recording above the service's file limit stays local (with its poster and preview), the rest is shared
1400
+ const tooBig = new Set(entries.filter((e) => /\.rec\./.test(e.path) && u?.limits?.max_file_bytes && e.bytes > u.limits.max_file_bytes).map((e) => e.path))
1401
+ if (tooBig.size) {
1402
+ manifest = withoutRecordings(manifest, tooBig)
1403
+ const keep = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
1404
+ entries = entries.filter((e) => !/\.rec\./.test(e.path) || keep.has(e.path))
1405
+ say(`${tooBig.size} recording(s) larger than ${fmtBytes(u.limits.max_file_bytes)} stay in the local preview: ${[...tooBig].slice(0, 3).join(', ')}${tooBig.size > 3 ? ', …' : ''}. Record them with record.quality: low or a lower max_seconds to share them.`)
1406
+ }
1330
1407
  const missing = await missingHashes(client, config.org, entries.map((e) => e.hash))
1331
1408
  const total = entries.reduce((n, e) => n + e.bytes, 0)
1332
1409
  const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.bytes])).values()].reduce((n, b) => n + b, 0) : total
1333
1410
  const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
1334
1411
  try {
1335
- const u = await client.request('GET', `/orgs/${config.org}/usage`)
1412
+ if (!u) throw new Error('usage unavailable')
1336
1413
  if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}. Capture fewer viewports or locales, or ask an admin for a higher limit.`)
1337
1414
  if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
1338
- say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
1415
+ const recBytes = entries.filter((e) => /\.rec\./.test(e.path)).reduce((n, e) => n + e.bytes, 0)
1416
+ say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${recBytes ? ` (recordings ${fmtBytes(recBytes)})` : ''}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
1339
1417
  } catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
1340
1418
  const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
1341
1419
  await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})