kaloko 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +1 -1
- package/schemas/config.v1.json +28 -0
- package/schemas/scenario.v1.json +23 -1
- package/skills/kaloko/SKILL.md +10 -3
- package/skills/kaloko/reference/mcp.md +1 -0
- package/skills/kaloko/reference/recording.md +95 -0
- package/src/cli.mjs +91 -13
- package/src/lib/anim.mjs +125 -0
- package/src/lib/apps/recording-native.mjs +212 -0
- package/src/lib/build.mjs +60 -4
- package/src/lib/doctor.mjs +5 -1
- package/src/lib/import.mjs +1 -1
- package/src/lib/recording.mjs +373 -0
- package/src/lib/trace.mjs +109 -0
- package/src/lib/video-cli.mjs +209 -0
- package/src/lib/video.mjs +427 -0
- package/src/lib/webm.mjs +144 -0
- package/viewer/index.html +186 -7
package/README.md
CHANGED
|
@@ -113,6 +113,24 @@ jobs:
|
|
|
113
113
|
Uploads go by content hash: captures that did not change since an earlier run
|
|
114
114
|
are not sent again, and an interrupted `share` picks up where it stopped.
|
|
115
115
|
|
|
116
|
+
## When a screenshot is not enough
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
npx kaloko walk --record # each step on video, from the previous screen to its capture
|
|
120
|
+
npx kaloko walk --record failed # only the steps that fail (flaky ones, mostly)
|
|
121
|
+
npx kaloko walk --record flow # the whole walk per language, a chapter per step
|
|
122
|
+
npx kaloko video export --format mp4 # the run as one video, a title card and verdict before each step
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The canvas plays a step's video next to its screenshot, and the Technical tab
|
|
126
|
+
shows its trace: clicks, typing, requests with status and timing, console
|
|
127
|
+
messages. Click a row and the video jumps there. Web recordings need nothing
|
|
128
|
+
but Chrome (WebM with VP9); with ffmpeg installed you also get MP4 for Safari,
|
|
129
|
+
app recordings (Android `screenrecord`, iOS Simulator, macOS) and the
|
|
130
|
+
walkthrough export. `npx kaloko video tools` says what this machine has.
|
|
131
|
+
Recordings stay in the local preview on Free and Team; Business and Enterprise
|
|
132
|
+
share them with the run.
|
|
133
|
+
|
|
116
134
|
## Screenshots you already have
|
|
117
135
|
|
|
118
136
|
```bash
|
|
@@ -193,6 +211,7 @@ Nothing in Kaloko is stored in a format only Kaloko can read. Every export opens
|
|
|
193
211
|
| Live files (handoff) | `npx kaloko export --live [--version n] --out design/` | the design folder as written |
|
|
194
212
|
| Shared run | `npx kaloko export --run <id>` | zip with `run.json` (`qawalk.run.v1`), screenshots and HTML snapshots, live HTML of design steps, comments and verdicts, and `index.html`: the canvas, offline |
|
|
195
213
|
| Documentation captures | `npx kaloko export --scenario <id>` | newest image and HTML of every step, `index.json` |
|
|
214
|
+
| Walkthrough video | `npx kaloko video export [--format mp4\|webm\|gif]` | one video of the run's recordings with title cards; step recordings are WebM (VP9) and MP4 (H.264), traces JSON (`qawalk.trace.v1`) |
|
|
196
215
|
|
|
197
216
|
The schemas of these formats ship with the package (`schemas/`).
|
|
198
217
|
|
package/package.json
CHANGED
package/schemas/config.v1.json
CHANGED
|
@@ -81,6 +81,15 @@
|
|
|
81
81
|
"concurrency": { "type": "integer", "minimum": 1, "maximum": 16, "description": "Locales walked at the same time, each in its own browser context (kaloko walk --concurrency). Default: by CPU in an own browser, one at a time on the shared browser." }
|
|
82
82
|
}
|
|
83
83
|
},
|
|
84
|
+
"capture": {
|
|
85
|
+
"type": "object",
|
|
86
|
+
"additionalProperties": false,
|
|
87
|
+
"description": "Defaults for every scenario of the project; a scenario's capture block wins.",
|
|
88
|
+
"properties": {
|
|
89
|
+
"record": { "$ref": "#/$defs/record" },
|
|
90
|
+
"trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
|
|
91
|
+
}
|
|
92
|
+
},
|
|
84
93
|
"share": {
|
|
85
94
|
"type": "object",
|
|
86
95
|
"additionalProperties": false,
|
|
@@ -92,6 +101,25 @@
|
|
|
92
101
|
}
|
|
93
102
|
},
|
|
94
103
|
"$defs": {
|
|
104
|
+
"record": {
|
|
105
|
+
"description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
|
|
106
|
+
"oneOf": [
|
|
107
|
+
{ "type": "boolean" },
|
|
108
|
+
{ "enum": ["step", "flow", "failed", "off"] },
|
|
109
|
+
{
|
|
110
|
+
"type": "object",
|
|
111
|
+
"additionalProperties": false,
|
|
112
|
+
"properties": {
|
|
113
|
+
"mode": { "enum": ["step", "flow", "failed", "off"], "default": "step" },
|
|
114
|
+
"quality": { "enum": ["low", "medium", "high"], "default": "medium", "description": "low 8 fps up to 960 px, medium 12 fps up to 1280 px, high 24 fps up to 1920 px" },
|
|
115
|
+
"max_seconds": { "type": "integer", "minimum": 1, "maximum": 3600, "description": "Longest kept recording; a longer one keeps its end (default 60 for a step, 900 for a flow)" },
|
|
116
|
+
"codec": { "enum": ["auto", "vp9", "av1", "h264"], "default": "auto", "description": "auto: WebM VP9 (VP8 fallback) from the browser, plus MP4 H.264 when ffmpeg is installed; av1 needs ffmpeg" },
|
|
117
|
+
"mp4": { "oneOf": [{ "type": "boolean" }, { "const": "auto" }], "default": "auto", "description": "MP4 H.264 next to the WebM (Safari before iOS 17.4); needs ffmpeg" },
|
|
118
|
+
"cursor": { "type": "boolean", "default": true, "description": "Draw the pointer and clicks into web recordings" }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
]
|
|
122
|
+
},
|
|
95
123
|
"environment": {
|
|
96
124
|
"type": "object",
|
|
97
125
|
"additionalProperties": false,
|
package/schemas/scenario.v1.json
CHANGED
|
@@ -163,6 +163,25 @@
|
|
|
163
163
|
}
|
|
164
164
|
},
|
|
165
165
|
"$defs": {
|
|
166
|
+
"record": {
|
|
167
|
+
"description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
|
|
168
|
+
"oneOf": [
|
|
169
|
+
{ "type": "boolean" },
|
|
170
|
+
{ "enum": ["step", "flow", "failed", "off"] },
|
|
171
|
+
{
|
|
172
|
+
"type": "object",
|
|
173
|
+
"additionalProperties": false,
|
|
174
|
+
"properties": {
|
|
175
|
+
"mode": { "enum": ["step", "flow", "failed", "off"], "default": "step" },
|
|
176
|
+
"quality": { "enum": ["low", "medium", "high"], "default": "medium", "description": "low 8 fps up to 960 px, medium 12 fps up to 1280 px, high 24 fps up to 1920 px" },
|
|
177
|
+
"max_seconds": { "type": "integer", "minimum": 1, "maximum": 3600, "description": "Longest kept recording; a longer one keeps its end (default 60 for a step, 900 for a flow)" },
|
|
178
|
+
"codec": { "enum": ["auto", "vp9", "av1", "h264"], "default": "auto", "description": "auto: WebM VP9 (VP8 fallback) from the browser, plus MP4 H.264 when ffmpeg is installed; av1 needs ffmpeg" },
|
|
179
|
+
"mp4": { "oneOf": [{ "type": "boolean" }, { "const": "auto" }], "default": "auto", "description": "MP4 H.264 next to the WebM (Safari before iOS 17.4); needs ffmpeg" },
|
|
180
|
+
"cursor": { "type": "boolean", "default": true, "description": "Draw the pointer and clicks into web recordings" }
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
]
|
|
184
|
+
},
|
|
166
185
|
"slug": {
|
|
167
186
|
"type": "string",
|
|
168
187
|
"pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
|
|
@@ -269,6 +288,7 @@
|
|
|
269
288
|
"capture": {
|
|
270
289
|
"$ref": "#/$defs/capture"
|
|
271
290
|
},
|
|
291
|
+
"record": { "$ref": "#/$defs/record", "description": "Record this step even when the walk does not (true), or never (false)." },
|
|
272
292
|
"ignore": {
|
|
273
293
|
"type": "array",
|
|
274
294
|
"description": "Regions left out of pixel comparisons (clocks, ads, carousels): a selector, or a rectangle in page pixels of one viewport. kaloko stability suggests them.",
|
|
@@ -710,7 +730,9 @@
|
|
|
710
730
|
"full_page": {
|
|
711
731
|
"type": "boolean",
|
|
712
732
|
"description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
|
|
713
|
-
}
|
|
733
|
+
},
|
|
734
|
+
"record": { "$ref": "#/$defs/record" },
|
|
735
|
+
"trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
|
|
714
736
|
}
|
|
715
737
|
}
|
|
716
738
|
}
|
package/skills/kaloko/SKILL.md
CHANGED
|
@@ -13,7 +13,8 @@ description: >
|
|
|
13
13
|
design environment (live HTML steps, drafts, versions, the design
|
|
14
14
|
reference, checking implementation against it), and when implementing a
|
|
15
15
|
screen that has an approved design (read the step spec, build with its
|
|
16
|
-
tokens, check drift)
|
|
16
|
+
tokens, check drift), and when a step should be recorded on video or its
|
|
17
|
+
trace read (actions, requests, console).
|
|
17
18
|
---
|
|
18
19
|
|
|
19
20
|
# Kaloko
|
|
@@ -50,7 +51,9 @@ create test data there (users, roles, carts), capture screens and e-mails,
|
|
|
50
51
|
evaluate, build the preview, and fix scenario mechanics (selectors, scripts)
|
|
51
52
|
that do not change what is being accepted, including applying a selector
|
|
52
53
|
proposal with `kaloko walk --heal` when it names the same element. Walk only
|
|
53
|
-
the steps a change reaches (`--affected`) and import screenshots into a run.
|
|
54
|
+
the steps a change reaches (`--affected`) and import screenshots into a run. Record steps locally
|
|
55
|
+
(`walk --record`) when a screenshot cannot show what a person has to judge, read traces, and
|
|
56
|
+
export a walkthrough video into the run folder. On a design environment: change
|
|
54
57
|
the design folder, walk, and sync the draft (including taking others' steps
|
|
55
58
|
with `--theirs`). Move to the trash runs you uploaded in this session that
|
|
56
59
|
were superseded before anyone looked at them (debug rounds: no comments, no
|
|
@@ -62,7 +65,8 @@ Changing acceptance criteria or scope, any production run, writing or
|
|
|
62
65
|
replacing a CI workflow (`kaloko ci github|gitlab`), uploading with
|
|
63
66
|
`kaloko share` (on a design environment: publishing a version), setting a
|
|
64
67
|
design reference or baseline, merging a design branch, keeping your version of
|
|
65
|
-
a step in a sync conflict (`--mine`), turning on `html_view`,
|
|
68
|
+
a step in a sync conflict (`--mine`), turning on `html_view`, turning on recordings for
|
|
69
|
+
a whole project (`capture.record` in the config: storage), a real clean-up
|
|
66
70
|
(`kaloko prune --yes`), trashing someone else's run, archiving a scenario, and
|
|
67
71
|
anything the environment profile blocks. Never trash an accepted run, a
|
|
68
72
|
baseline, a run with review comments, or a run linked from an open PR or issue
|
|
@@ -93,6 +97,9 @@ they can do; never write secrets into the config, a scenario or a script.
|
|
|
93
97
|
(`--heal`): [reference/execution.md](reference/execution.md)
|
|
94
98
|
- Screenshots, walkthroughs, Playwright reports and traces that an agent or a
|
|
95
99
|
tool already produced, laid out as a run: [reference/import.md](reference/import.md)
|
|
100
|
+
- Video of a step or a whole flow, traces (actions, requests, console) and the
|
|
101
|
+
walkthrough video; when recording is worth its storage:
|
|
102
|
+
[reference/recording.md](reference/recording.md)
|
|
96
103
|
- CI: a workflow that walks each pull request's preview deployment and
|
|
97
104
|
comments on it (`kaloko ci github|gitlab`): [reference/ci.md](reference/ci.md)
|
|
98
105
|
- Evaluation, JEV thresholds and reading results: [reference/evaluation.md](reference/evaluation.md)
|
|
@@ -28,6 +28,7 @@ file.
|
|
|
28
28
|
| `list_notifications` | the token owner's inbox (`unread_only`, `limit`) |
|
|
29
29
|
| `list_projects` | projects of the organization with counts; the slug is the `project` filter of `list_runs` and the value for `project:` in a scenario |
|
|
30
30
|
| `get_criteria_verdicts` | reviewer verdicts on manual or overridden criteria of a run |
|
|
31
|
+
| `get_step_recording` | a step's recording and trace (`run_id`, `step`, `locale`, `viewport`, `trace: true` for the full trace): video URLs, duration, its chapter of a flow recording, failed attempts, the trace summary (failed requests, console errors, slowest request); see [recording.md](recording.md). `get_run_summary` lists trace summaries per step too |
|
|
31
32
|
| `get_review_queue` | what a person still has to decide in a run (`run_id`, or `scenario` / nothing for the newest run): steps in review order with reasons, decisions, approvals carried over unchanged steps with their origin, counts and `review_url` to give the person |
|
|
32
33
|
| `decide_step` | records your (the agent's) decision on a step: `approve`, or `return` with a `note`; marked as an agent's and not counted as a person's. Never use it to approve for a person |
|
|
33
34
|
| `revoke_carried_approval` | revokes an approval carried over to a step that looks unchanged but should be looked at again (`run_id`, `step`) |
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# Recordings, traces and the walkthrough video
|
|
2
|
+
|
|
3
|
+
A screenshot shows where a step ended. A recording shows how it got there:
|
|
4
|
+
the clicks, the spinner that never stopped, the menu that opened twice. A trace
|
|
5
|
+
lists what happened meanwhile: actions, network requests with status, size and
|
|
6
|
+
timing, and console messages.
|
|
7
|
+
|
|
8
|
+
## When to record
|
|
9
|
+
|
|
10
|
+
Record when a screenshot cannot show the problem or the behaviour a reviewer
|
|
11
|
+
has to judge:
|
|
12
|
+
|
|
13
|
+
- a flaky step (passes, then fails on the same commit): `--record failed` keeps
|
|
14
|
+
only the attempts that fail;
|
|
15
|
+
- animations, transitions, loading states, drag and drop, menus and dialogs
|
|
16
|
+
that open and close;
|
|
17
|
+
- a bug report or a PR where the person needs to see the interaction once;
|
|
18
|
+
- a demo for a client or a release note (`kaloko video export`).
|
|
19
|
+
|
|
20
|
+
Do not record every walk by default. Recordings are much larger than
|
|
21
|
+
screenshots, count in the organization's storage and expire after 30 days on
|
|
22
|
+
Business. Traces are small and are kept for every walked web step anyway.
|
|
23
|
+
|
|
24
|
+
## Commands
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
kaloko walk --record # each step: from the previous screen to its capture
|
|
28
|
+
kaloko walk --record flow # the whole walk per locale, a chapter per step
|
|
29
|
+
kaloko walk --record failed # only steps whose script fails or whose checks fail
|
|
30
|
+
kaloko walk --record --record-quality low --steps checkout
|
|
31
|
+
kaloko walk --no-trace # no trace either
|
|
32
|
+
kaloko video list # what was recorded and traced, with sizes
|
|
33
|
+
kaloko video trace --step checkout # the step's trace as a timeline in the terminal
|
|
34
|
+
kaloko video export --format mp4 # the run's recordings with a title card per step
|
|
35
|
+
kaloko video tools # ffmpeg, adb, simctl on this machine
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
In a scenario, or in `kaloko.config.yml` for the whole project:
|
|
39
|
+
|
|
40
|
+
```yaml
|
|
41
|
+
capture:
|
|
42
|
+
record: { mode: failed, quality: low, max_seconds: 45 } # or true, step, flow, failed, false
|
|
43
|
+
trace: true
|
|
44
|
+
nodes:
|
|
45
|
+
- id: checkout
|
|
46
|
+
record: true # this step always; record: false never
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Quality presets: `low` 8 fps up to 960 px, `medium` (default) 12 fps up to
|
|
50
|
+
1280 px, `high` 24 fps up to 1920 px. A step keeps its last 60 seconds and a
|
|
51
|
+
flow its last 15 minutes unless `max_seconds` says otherwise.
|
|
52
|
+
|
|
53
|
+
## What you get
|
|
54
|
+
|
|
55
|
+
Per recorded step under `steps/<step>/<locale>/<viewport>.`: `rec.webm` (VP9
|
|
56
|
+
from Chrome, no extra tools), `rec.mp4` (H.264, when ffmpeg is installed; Safari
|
|
57
|
+
before iOS 17.4 needs it), `rec.poster.webp`, `rec.webp` (a few-second animated
|
|
58
|
+
preview; `rec.gif` where the browser cannot encode WebP) and `trace.json`. The
|
|
59
|
+
pointer and clicks are drawn into web recordings. A flow recording is
|
|
60
|
+
`attachments/flow/<locale>.rec.*`; the walkthrough is
|
|
61
|
+
`attachments/walkthrough.rec.<format>`.
|
|
62
|
+
|
|
63
|
+
Web and Electron record through Chrome's screencast; walks in Firefox or WebKit
|
|
64
|
+
are neither recorded nor traced. Apps: Android with `adb shell screenrecord`
|
|
65
|
+
(segments, touches shown), the iOS Simulator with `simctl io recordVideo`,
|
|
66
|
+
macOS desktop apps with `screencapture` (Screen Recording permission). App
|
|
67
|
+
recordings need ffmpeg for the poster and preview; `kaloko doctor` lists what is
|
|
68
|
+
missing with the install command for the OS.
|
|
69
|
+
|
|
70
|
+
## On the canvas
|
|
71
|
+
|
|
72
|
+
The step detail gets Video next to Screenshot: the poster first, nothing loads
|
|
73
|
+
until played, markers for the start, the capture, actions and failed requests.
|
|
74
|
+
Technical shows the trace; a click on a row moves the video there. Cards play
|
|
75
|
+
the preview on hover. Replay plays the flow recording with its chapters. The
|
|
76
|
+
walkthrough video is in the top bar.
|
|
77
|
+
|
|
78
|
+
## Plans and sharing
|
|
79
|
+
|
|
80
|
+
Recording is local and works on every plan. `kaloko share` uploads recordings
|
|
81
|
+
on Business and Enterprise; on Free and Team it says they stay in the local
|
|
82
|
+
preview and shares screenshots and traces. A recording over the service's file
|
|
83
|
+
limit (25 MB) stays local too: lower the quality or `max_seconds`. Recordings
|
|
84
|
+
expire 30 days after the run is shared (Enterprise chooses in Settings →
|
|
85
|
+
Storage); the screenshots stay as long as the run. Uploading is still "ask
|
|
86
|
+
first", and so is turning recordings on for a whole project.
|
|
87
|
+
|
|
88
|
+
## Reading them as an agent
|
|
89
|
+
|
|
90
|
+
`kaloko video trace --step <id>` locally, or MCP `get_step_recording`
|
|
91
|
+
(`run_id`, `step`; `trace: true` adds the full trace) on a shared run. It
|
|
92
|
+
answers the video URLs, duration, the step's chapter of a flow recording,
|
|
93
|
+
failed attempts and the trace summary: failed requests with status, console
|
|
94
|
+
errors, the slowest request. Fetch the URLs with the same token. Start there
|
|
95
|
+
when a step failed and the evidence in the criteria is not enough.
|
package/src/cli.mjs
CHANGED
|
@@ -51,6 +51,9 @@ import { exportDesign } from './lib/design-export.mjs'
|
|
|
51
51
|
import { driverFor, isApp, pngSize, ocr as ocrImage } from './lib/apps/drivers.mjs'
|
|
52
52
|
import { captureApp } from './lib/apps/capture.mjs'
|
|
53
53
|
import { readTree, flatten } from './lib/apps/tree.mjs'
|
|
54
|
+
import { recordSettings, recordingSession, recordingFiles, traceFiles, stripRecordings, withoutRecordings } from './lib/recording.mjs'
|
|
55
|
+
import { nativeSession } from './lib/apps/recording-native.mjs'
|
|
56
|
+
import { videoCmd } from './lib/video-cli.mjs'
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
const HELP = `kaloko — walk, assess and approve what your agents built, on one flow canvas
|
|
@@ -130,6 +133,11 @@ Run
|
|
|
130
133
|
steps' captures from the previous local run
|
|
131
134
|
[--heal | --no-heal] A selector the page lost gets a proposal from the page (test id, role and name,
|
|
132
135
|
text) with a patch; --heal applies it to the step script once the step passes
|
|
136
|
+
[--record [step|flow|failed]] [--record-quality low|medium|high] [--no-trace]
|
|
137
|
+
Record each step from the previous screen to its capture (step), the whole walk
|
|
138
|
+
per locale with a chapter per step (flow), or only steps that fail (failed);
|
|
139
|
+
also capture.record in the config or scenario, record: true on a step.
|
|
140
|
+
A trace (actions, requests, console) is kept per step unless --no-trace
|
|
133
141
|
affected [--scenario file | --all] [--base ref] [--files a,b] [--json] [--check] [--offline]
|
|
134
142
|
Which steps changed files reach, and why: framework route files against step paths
|
|
135
143
|
and URLs captured earlier, imports, global files, i18n messages, step scripts and
|
|
@@ -147,6 +155,11 @@ Results
|
|
|
147
155
|
the semantic questions again on the stored captures (no walk);
|
|
148
156
|
console, overflow and performance keep their live result
|
|
149
157
|
build Write run.json and the local preview (index.html)
|
|
158
|
+
video export [--format mp4|webm|gif] [--locale l] [--steps a,b] [--lang en|cs] [--out file]
|
|
159
|
+
The run's recordings in flow order with a title card per step (title, verdict);
|
|
160
|
+
shared with the run on Business and up
|
|
161
|
+
video list | trace --step <id> | tools
|
|
162
|
+
Recordings and traces of the run; a step's trace as a timeline; ffmpeg, adb, simctl
|
|
150
163
|
preview Build and print the preview path
|
|
151
164
|
share [--expires 30] [--pr] Upload the run to kaloko.app (KALOKO_TOKEN, tester role); files travel by content
|
|
152
165
|
hash, so unchanged captures are not sent again and a broken-off share resumes;
|
|
@@ -505,6 +518,14 @@ function linkFor (env, node, locale) {
|
|
|
505
518
|
if (/^[a-z][a-z0-9+.-]*:/i.test(p)) return p
|
|
506
519
|
return env.base_url ? urlFor(env, { path: p }, locale) : null
|
|
507
520
|
}
|
|
521
|
+
/** A headless Chrome of its own for encoding recordings, started on first use (one per walk). */
|
|
522
|
+
function lazyEncoder (config) {
|
|
523
|
+
let started = null
|
|
524
|
+
return {
|
|
525
|
+
page: () => (started ||= launchEphemeral(config).then(async (b) => ({ b, p: await b.newPage() })).catch(() => ({ b: null, p: null }))).then((x) => x.p),
|
|
526
|
+
close: async () => { if (started) await (await started).b?.close().catch(() => {}) }
|
|
527
|
+
}
|
|
528
|
+
}
|
|
508
529
|
const appViewport = (run) => Object.keys(run.meta.scope.viewports || {})[0] || null
|
|
509
530
|
|
|
510
531
|
/**
|
|
@@ -522,6 +543,8 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
|
522
543
|
const encoder = enc ? await enc.newPage().catch(() => null) : null
|
|
523
544
|
const streak = failureStreak(args['keep-going'] ? 0 : 3)
|
|
524
545
|
const maxChars = config.evaluators?.jev?.max_state_chars
|
|
546
|
+
// recordings and traces (batch D): Electron windows use the screencast recorder, devices record themselves
|
|
547
|
+
const recSettings = recordSettings({ args, config, scenario })
|
|
525
548
|
try {
|
|
526
549
|
for (const locale of locales) {
|
|
527
550
|
const driver = driverFor(platform, env, { root, scenarioDir, locale })
|
|
@@ -530,6 +553,9 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
|
530
553
|
if (driver.web) observeConsole(driver.page)
|
|
531
554
|
say(`${platform} ${locale}: ${driver.web ? 'Electron window' : 'app'} ready`)
|
|
532
555
|
} catch (error) { failed.push(`${platform} (${locale}): ${error.message.split('\n')[0]}`); say(`✗ ${platform} ${locale}: ${error.message.split('\n')[0]}`); await driver.close().catch(() => {}); if (error instanceof UserError) break; continue }
|
|
556
|
+
const rec = driver.web
|
|
557
|
+
? await recordingSession({ page: driver.page, run, scenario, locale, settings: recSettings, viewports: run.meta.scope.viewports, encoder: async () => encoder, say, label: `${platform} ${locale}` }).catch(() => null)
|
|
558
|
+
: await nativeSession({ platform, env, run, scenario, locale, settings: recSettings, driver, encoder: async () => encoder, say }).catch(() => null)
|
|
533
559
|
try {
|
|
534
560
|
for (const node of scenario.nodes) {
|
|
535
561
|
if (node.kind === 'decision' || (pick && !pick.includes(node.id))) continue
|
|
@@ -542,30 +568,42 @@ async function walkApp ({ config, root, run, scenario, env, args }) {
|
|
|
542
568
|
const link = linkFor(env, node, locale)
|
|
543
569
|
if (!scripted && !(link && ['playwright', 'script'].includes(node.execution?.strategy))) { pending.push(`${node.id} (${locale}): ${node.execution?.strategy || 'agent'} — ${(node.execution?.instructions || '').split('\n')[0]}`); continue }
|
|
544
570
|
if (streak.stopped(locale)) { skipped.push(`${node.id} (${locale})`); continue }
|
|
571
|
+
const records = []
|
|
545
572
|
try {
|
|
546
573
|
let captured = false
|
|
574
|
+
await rec?.beforeStep(node)
|
|
547
575
|
const capture = async (o = {}) => {
|
|
548
576
|
captured = true
|
|
577
|
+
await rec?.beforeCapture(node)
|
|
549
578
|
if (driver.web) {
|
|
550
579
|
const [w, h] = await driver.page.evaluate(() => [innerWidth, innerHeight])
|
|
551
|
-
|
|
580
|
+
const r = await captureStep({ page: driver.page, node, locale, options: captureOptions(scenario, node), viewports: { [o.viewport || appViewport(run) || 'window']: [w, h] }, run, consoleFile: null, maxChars, extra: { platform, device: 'Electron' } })
|
|
581
|
+
records.push(...r)
|
|
582
|
+
return r
|
|
552
583
|
}
|
|
553
|
-
|
|
584
|
+
const r = await captureApp({ snapshot: await driver.snapshot(), node, locale, viewport: o.viewport || appViewport(run), run, platform, encoder, maxChars, extra: { url: link } })
|
|
585
|
+
records.push(r)
|
|
586
|
+
return r
|
|
554
587
|
}
|
|
555
588
|
if (scripted) {
|
|
556
589
|
const mod = await import(pathToFileURL(resolve(scenarioDir, node.execution.script)).href)
|
|
557
590
|
await mod.default({ app: driver, page: driver.page || null, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, env, run: run.meta, capture, link: (id = node.id) => linkFor(env, stepOf(scenario, id), locale), url: (id = node.id) => linkFor(env, stepOf(scenario, id), locale) })
|
|
558
591
|
} else await driver.link(link)
|
|
559
592
|
if (!captured) await capture()
|
|
593
|
+
await rec?.afterStep(node, driver.web ? { records } : { record: records[0] })
|
|
560
594
|
say(`✓ ${node.id} ${locale}`)
|
|
561
595
|
streak.ok(locale)
|
|
562
596
|
} catch (error) {
|
|
597
|
+
await rec?.afterStep(node, driver.web ? { error, records } : { error, record: records[0] }).catch(() => {})
|
|
563
598
|
failed.push(`${node.id} (${locale}): ${error.message.split('\n')[0]}`)
|
|
564
599
|
say(`✗ ${node.id} ${locale}: ${error.message.split('\n')[0]}`)
|
|
565
600
|
if (streak.record(locale, error.message)) say(` stopping ${locale}: the last 3 steps failed with the same error`)
|
|
566
601
|
}
|
|
567
602
|
}
|
|
568
|
-
} finally {
|
|
603
|
+
} finally {
|
|
604
|
+
for (const n of (await rec?.finish().catch((e) => [`recording: ${e.message}`])) || []) say(` ${n}`)
|
|
605
|
+
await driver.close().catch(() => {})
|
|
606
|
+
}
|
|
569
607
|
}
|
|
570
608
|
} finally { await enc?.close().catch(() => {}) }
|
|
571
609
|
if (pending.length) say(`\nSteps for an agent or a person (drive the app, then kaloko capture --step <id> --image shot.png [--tree dump.xml|json]):\n - ${[...new Set(pending)].join('\n - ')}`)
|
|
@@ -861,6 +899,11 @@ const commands = {
|
|
|
861
899
|
say(`✓ ${node.id} ${locale}: "${results.subject}" → ${results.count} viewport(s)`)
|
|
862
900
|
},
|
|
863
901
|
|
|
902
|
+
async video (args) {
|
|
903
|
+
const { config, root } = await context(args)
|
|
904
|
+
await videoCmd({ config, root, args, say })
|
|
905
|
+
},
|
|
906
|
+
|
|
864
907
|
async walk (args) {
|
|
865
908
|
const { config, root, run, scenario, env } = await runContext(args)
|
|
866
909
|
if (args.affected) {
|
|
@@ -905,6 +948,9 @@ const commands = {
|
|
|
905
948
|
// semantic questions of every new capture start right away; the walk ends with the verdicts (--no-evaluate: skip)
|
|
906
949
|
const prefetch = args['no-evaluate'] ? null : semanticPrefetch({ scenario, config, root })
|
|
907
950
|
const walked = await readCaptures(run.dir)
|
|
951
|
+
// recordings and traces (batch D, #12): opt-in recordings, traces unless turned off; encoded in an own headless Chrome
|
|
952
|
+
const recSettings = recordSettings({ args, config, scenario })
|
|
953
|
+
const encoder = lazyEncoder(config)
|
|
908
954
|
const sessions = scenario.sessions || null
|
|
909
955
|
// named sessions: one isolated browser context per person and locale (cookies, storage, sign-in), created lazily
|
|
910
956
|
const sessionContexts = new Map()
|
|
@@ -939,6 +985,12 @@ const commands = {
|
|
|
939
985
|
const page = await lctx.newPage()
|
|
940
986
|
const shared = conn && !parallel // the shared browser's own context logs its console to a file
|
|
941
987
|
if (!shared) observeConsole(page)
|
|
988
|
+
// one recorder per page (the walk's tab and each named session's), started on its first step
|
|
989
|
+
const recorders = new Map()
|
|
990
|
+
const recorderFor = async (p) => {
|
|
991
|
+
if (!recorders.has(p)) recorders.set(p, recordingSession({ page: p, run, scenario, locale, settings: recSettings, viewports: run.meta.scope.viewports, encoder: encoder.page, say, label: locale }).catch(() => null))
|
|
992
|
+
return recorders.get(p)
|
|
993
|
+
}
|
|
942
994
|
try {
|
|
943
995
|
// every locale starts as a new visitor on its first walk in the run (an own context is new already)
|
|
944
996
|
if (shared && !walked.some((c) => c.locale === locale)) { await resetSession(lctx, page, baseUrl(env, locale)); say(`session reset (new visitor, ${locale})`) }
|
|
@@ -978,11 +1030,17 @@ const commands = {
|
|
|
978
1030
|
const scriptFile = resolve(scenarioDir, node.execution.script)
|
|
979
1031
|
// one invocation of the step script; the automatic capture runs unless the script captured the step itself:
|
|
980
1032
|
// a crop is an attachment (#49), and a capture from an earlier walk must not stand in for this one (#50)
|
|
1033
|
+
const rec = await recorderFor(sPage)
|
|
981
1034
|
const runStep = async (mod) => {
|
|
982
1035
|
let captured = false
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
1036
|
+
rec?.beforeStep(node)
|
|
1037
|
+
const records = []
|
|
1038
|
+
const capture = (opts = {}) => { captured = true; rec?.beforeCapture(node); return captureStep({ page: sPage, node, locale, options: captureOptions(scenario, node), viewports: scopeViewports(run, opts.viewports), run, htmlView: htmlViewAllowed(env, scenario) ? { scenarioId: scenario.id } : null, consoleFile, maxChars: config.evaluators?.jev?.max_state_chars, extra: node.session ? { session: node.session } : {}, stability: Number(args.stability) || 0 }).then((r) => { prefetch?.add(r, run.dir); records.push(...r); return r }) }
|
|
1039
|
+
try {
|
|
1040
|
+
await mod.default({ page: sPage, context: sCtx, locale, node: { id: node.id, title: localized(node.title, locale), kind: node.kind }, session: sessionInfo(node.session), url: (id = node.id) => urlFor(env, stepOf(scenario, id), locale), env, run: run.meta, capture, crop: (selector, title, o = {}) => captureCrop({ page: sPage, node, locale, selector, title, run, padding: Number(o.padding) || 0 }), acceptDialogs: () => acceptDialogs(sPage) })
|
|
1041
|
+
if (!captured) await capture()
|
|
1042
|
+
} catch (error) { await rec?.afterStep(node, { error }); throw error }
|
|
1043
|
+
await rec?.afterStep(node, { records })
|
|
986
1044
|
}
|
|
987
1045
|
try {
|
|
988
1046
|
await runStep(await import(pathToFileURL(scriptFile).href))
|
|
@@ -1002,7 +1060,10 @@ const commands = {
|
|
|
1002
1060
|
if (streak.record(streakKey, error.message)) say(` stopping ${node.session ? `session ${node.session}` : 'this walk'} (${locale}): the last 3 steps failed with the same error; fix the shared cause (sign-in, environment) and rerun, or pass --keep-going`)
|
|
1003
1061
|
}
|
|
1004
1062
|
}
|
|
1005
|
-
} finally {
|
|
1063
|
+
} finally {
|
|
1064
|
+
for (const r of recorders.values()) { const notes = await (await r)?.finish().catch((e) => [`recording: ${e.message}`]); for (const n of notes || []) say(` ${n}`) }
|
|
1065
|
+
await page.close().catch(() => {})
|
|
1066
|
+
}
|
|
1006
1067
|
return { pending, failed, skipped }
|
|
1007
1068
|
}
|
|
1008
1069
|
let results
|
|
@@ -1014,6 +1075,7 @@ const commands = {
|
|
|
1014
1075
|
for (const c of ownContexts) await c.close().catch(() => {})
|
|
1015
1076
|
if (conn) await conn.browser.close() // over CDP this only disconnects; the shared browser keeps running
|
|
1016
1077
|
if (ephemeral) await ephemeral.close()
|
|
1078
|
+
await encoder.close()
|
|
1017
1079
|
await releaseLock?.()
|
|
1018
1080
|
}
|
|
1019
1081
|
const pending = results.flatMap((r) => r.pending); const failed = results.flatMap((r) => r.failed); const skipped = results.flatMap((r) => r.skipped)
|
|
@@ -1107,7 +1169,7 @@ const commands = {
|
|
|
1107
1169
|
const kept = await keepVideos(run, source.videos)
|
|
1108
1170
|
say(`\nImported ${captured} screen(s)${attached ? `, ${attached} more as attachments` : ''} into ${run.meta.id}.${encoder ? '' : ' (No Chrome: images kept as they are, without thumbnails.)'}`)
|
|
1109
1171
|
if (unmatched.length) say(`Not matched to a step (${unmatched.length}): name the step in steps.json ("step": "<id>"), or pass --map "${unmatched[0]}=<step id>". Steps: ${scenario.nodes.filter((n) => n.kind !== 'decision').map((n) => n.id).join(', ')}`)
|
|
1110
|
-
if (kept.length) say(`Recordings kept in the run folder, not
|
|
1172
|
+
if (kept.length) say(`Recordings kept in the run folder, not attached to a step and not uploaded: ${kept.join(', ')}. kaloko walk --record records steps for the canvas.`)
|
|
1111
1173
|
say('Next: kaloko evaluate, then kaloko preview (local canvas) or kaloko share. Imported screens have no page to check: their deterministic criteria wait for a walk, semantic ones for a person.')
|
|
1112
1174
|
},
|
|
1113
1175
|
|
|
@@ -1311,9 +1373,16 @@ const commands = {
|
|
|
1311
1373
|
await writeFile(join(run.dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n')
|
|
1312
1374
|
return
|
|
1313
1375
|
}
|
|
1314
|
-
|
|
1376
|
+
let manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
|
|
1315
1377
|
await writePreview({ run, manifest })
|
|
1316
|
-
|
|
1378
|
+
// the plan and storage before anything is sent; recordings travel on plans with the recordings feature only
|
|
1379
|
+
const u = await client.request('GET', `/orgs/${config.org}/usage`).catch(() => null)
|
|
1380
|
+
const recs = recordingFiles(manifest)
|
|
1381
|
+
if (recs.length && u?.limits?.features && !u.limits.features.recordings) {
|
|
1382
|
+
manifest = stripRecordings(manifest)
|
|
1383
|
+
say(`Recordings stay in the local preview (${recs.length} file(s)): sharing them is part of the Business plan and up, and this organization is on ${u.limits.plan_name}. Screenshots and traces are shared as usual.`)
|
|
1384
|
+
}
|
|
1385
|
+
const files = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
|
|
1317
1386
|
for (const n of manifest.nodes) {
|
|
1318
1387
|
for (const byVp of Object.values(n.captures || {})) for (const c of Object.values(byVp)) [c.image || c.png, c.thumb, c.html, c.eval, c.tree, c.spec].filter(Boolean).forEach((p) => files.add(p))
|
|
1319
1388
|
for (const a of n.attachments || []) files.add(a.file)
|
|
@@ -1326,16 +1395,25 @@ const commands = {
|
|
|
1326
1395
|
}
|
|
1327
1396
|
const days = Number(args.expires) || config.share?.expires_days || 30
|
|
1328
1397
|
// by content hash (#28): files the organization already stores (earlier runs, an interrupted share) are not sent
|
|
1329
|
-
|
|
1398
|
+
let entries = await hashFiles(run.dir, [...files])
|
|
1399
|
+
// a recording above the service's file limit stays local (with its poster and preview), the rest is shared
|
|
1400
|
+
const tooBig = new Set(entries.filter((e) => /\.rec\./.test(e.path) && u?.limits?.max_file_bytes && e.bytes > u.limits.max_file_bytes).map((e) => e.path))
|
|
1401
|
+
if (tooBig.size) {
|
|
1402
|
+
manifest = withoutRecordings(manifest, tooBig)
|
|
1403
|
+
const keep = new Set([...traceFiles(manifest), ...recordingFiles(manifest)])
|
|
1404
|
+
entries = entries.filter((e) => !/\.rec\./.test(e.path) || keep.has(e.path))
|
|
1405
|
+
say(`${tooBig.size} recording(s) larger than ${fmtBytes(u.limits.max_file_bytes)} stay in the local preview: ${[...tooBig].slice(0, 3).join(', ')}${tooBig.size > 3 ? ', …' : ''}. Record them with record.quality: low or a lower max_seconds to share them.`)
|
|
1406
|
+
}
|
|
1330
1407
|
const missing = await missingHashes(client, config.org, entries.map((e) => e.hash))
|
|
1331
1408
|
const total = entries.reduce((n, e) => n + e.bytes, 0)
|
|
1332
1409
|
const fresh = missing ? [...new Map(entries.filter((e) => missing.has(e.hash)).map((e) => [e.hash, e.bytes])).values()].reduce((n, b) => n + b, 0) : total
|
|
1333
1410
|
const resumed = (await readJson(join(run.dir, PROGRESS_FILE), null).catch(() => null))?.run === manifest.id
|
|
1334
1411
|
try {
|
|
1335
|
-
|
|
1412
|
+
if (!u) throw new Error('usage unavailable')
|
|
1336
1413
|
if (total > u.limits.max_run_bytes) throw new UserError(`This run is ${fmtBytes(total)}; the limit per run is ${fmtBytes(u.limits.max_run_bytes)}. Capture fewer viewports or locales, or ask an admin for a higher limit.`)
|
|
1337
1414
|
if (u.usage.bytes + fresh > u.limits.quota_bytes) throw new UserError(`Storage quota would be exceeded: ${fmtBytes(u.usage.bytes)} used + ${fmtBytes(fresh)} > ${fmtBytes(u.limits.quota_bytes)}. Delete old runs in Settings or ask for a higher quota.`)
|
|
1338
|
-
|
|
1415
|
+
const recBytes = entries.filter((e) => /\.rec\./.test(e.path)).reduce((n, e) => n + e.bytes, 0)
|
|
1416
|
+
say(`${resumed ? 'Resuming the interrupted share: ' : ''}${files.size} files, ${fmtBytes(total)}${recBytes ? ` (recordings ${fmtBytes(recBytes)})` : ''}${missing ? `; ${fmtBytes(fresh)} new, the rest already stored` : ''} (organization uses ${fmtBytes(u.usage.bytes)} of ${fmtBytes(u.limits.quota_bytes)})`)
|
|
1339
1417
|
} catch (e) { if (e instanceof UserError && /limit|quota/.test(e.message)) throw e }
|
|
1340
1418
|
const created = await client.request('POST', `/orgs/${config.org}/runs?expires_days=${days}`, manifest)
|
|
1341
1419
|
await writeFile(join(run.dir, PROGRESS_FILE), JSON.stringify({ run: manifest.id, started_at: new Date().toISOString() }) + '\n').catch(() => {})
|