@mrkt_frwd/reel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/docs/getting-started.md +280 -0
- package/package.json +24 -0
- package/src/actions.mjs +215 -0
- package/src/assemble.mjs +362 -0
- package/src/caption.mjs +115 -0
- package/src/capture.mjs +143 -0
- package/src/cli.mjs +318 -0
- package/src/clock.mjs +131 -0
- package/src/critic.mjs +271 -0
- package/src/cursor.mjs +118 -0
- package/src/edit-cli.mjs +211 -0
- package/src/edit.mjs +303 -0
- package/src/frame-grid.mjs +80 -0
- package/src/index.mjs +19 -0
- package/src/motion.mjs +229 -0
- package/src/runner.mjs +205 -0
- package/src/schema.mjs +157 -0
- package/src/server.mjs +109 -0
- package/src/timeline.mjs +120 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Joe Asare
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# @mrkt_frwd/reel
|
|
2
|
+
|
|
3
|
+
Drive a real browser from a declarative script and record it.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npx reel recordings/studio-tour/script.json --dry-run
|
|
7
|
+
npx reel recordings/studio-tour/script.json --deterministic
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
The script is data, not code. `--dry-run` validates it and resolves every
|
|
11
|
+
selector in seconds, so a capture never dies four minutes in on a renamed one.
|
|
12
|
+
`--deterministic` runs a fixed clock, so the frames do not depend on how long a
|
|
13
|
+
frame cost to draw.
|
|
14
|
+
|
|
15
|
+
Needs Chromium and ffmpeg — the only engine here that is not zero-dependency.
|
|
16
|
+
|
|
17
|
+
MIT.
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# Recording
|
|
2
|
+
|
|
3
|
+
Video of the product being used — multi-page flows, real interactions, WebGL captured as
|
|
4
|
+
painted, with an optional deterministic mode whose output does not depend on machine speed.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
npm run record:dry -- recordings/studio-tour/script.json # validate, resolve selectors
|
|
8
|
+
npm run record -- recordings/studio-tour/script.json # realtime capture
|
|
9
|
+
npm run record:exact -- recordings/studio-tour/script.json # deterministic capture
|
|
10
|
+
npm run record -- <script> --formats mp4,gif,vertical
|
|
11
|
+
npm run record -- <script> --live https://joeasare.com
|
|
12
|
+
npm run record -- <script> --keep-frames # keep frames for inspection
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Set `"capture": "deterministic"` in the script to make it the default for that recording.
|
|
16
|
+
|
|
17
|
+
`npm run record:test` validates every script in `recordings/` and runs inside `npm run gates`.
|
|
18
|
+
|
|
19
|
+
## Why a script file rather than browser code
|
|
20
|
+
|
|
21
|
+
A recording is `recordings/<name>/script.json`. It is declarative because more than one
|
|
22
|
+
agent touches it: a director writes the shots and their intent, an interaction developer
|
|
23
|
+
resolves the selectors and timings, a critic judges the frames against the stated intent.
|
|
24
|
+
A number in a JSON file can be diffed, reviewed and adjusted; a hundred lines of
|
|
25
|
+
imperative Playwright cannot be critiqued usefully by anyone who did not write it.
|
|
26
|
+
|
|
27
|
+
`name` must be a lowercase slug matching its directory. **Every shot needs an `intent`** —
|
|
28
|
+
the validator rejects a shot without one, because it is what a critique is measured
|
|
29
|
+
against.
|
|
30
|
+
|
|
31
|
+
## Architecture
|
|
32
|
+
|
|
33
|
+
| file | job |
|
|
34
|
+
|---|---|
|
|
35
|
+
| `packages/reel/src/cli.mjs` | entry point; validate, capture, encode, write `capture.json` |
|
|
36
|
+
| `packages/reel/src/runner.mjs` | owns the browser for the whole story; launch flags; one context |
|
|
37
|
+
| `packages/reel/src/actions.mjs` | the verb vocabulary, all eased |
|
|
38
|
+
| `packages/reel/src/capture.mjs` | CDP screencast to timestamped frames |
|
|
39
|
+
| `packages/reel/src/assemble.mjs` | ffmpeg: master MP4, 9:16 crop, GIF, critic samples |
|
|
40
|
+
| `packages/reel/src/cursor.mjs` | synthetic pointer, injected before page scripts |
|
|
41
|
+
| `packages/reel/src/clock.mjs` | the virtual clock shim for deterministic capture |
|
|
42
|
+
| `packages/reel/src/timeline.mjs` | the two timing drivers behind one `wait`/`animate` seam |
|
|
43
|
+
| `packages/reel/src/critic.mjs` | judges a finished recording from its frames |
|
|
44
|
+
| `packages/reel/src/edit.mjs` | edit decision lists — resolve segments against a capture |
|
|
45
|
+
| `packages/reel/src/edit-cli.mjs` | `npm run record:edit` |
|
|
46
|
+
| `packages/reel/src/caption.mjs` | captions rendered in Chromium, in studio typography |
|
|
47
|
+
| `packages/reel/src/frame-grid.mjs` | downsampled luminance grids for frame comparison |
|
|
48
|
+
| `packages/reel/src/schema.mjs` | validation and the dry run |
|
|
49
|
+
| `packages/reel/src/server.mjs` | static server honouring `cleanUrls`, with commerce endpoints stubbed |
|
|
50
|
+
|
|
51
|
+
**One browser context for the entire story.** That is what makes a multi-page narrative
|
|
52
|
+
work: cookies, `localStorage` and in-page state survive navigation, so a later shot can
|
|
53
|
+
show a state an earlier shot created. A recorder that opens a page per shot can only
|
|
54
|
+
produce disconnected clips.
|
|
55
|
+
|
|
56
|
+
**The cursor is drawn, not real.** The OS pointer is not part of what the compositor
|
|
57
|
+
paints, so a screencast of a click shows the effect and never the cause — fields appear to
|
|
58
|
+
fill themselves. `cursor.mjs` injects a pointer that follows real `pointermove` events and
|
|
59
|
+
draws a ripple on click. It is `pointer-events:none` and tagged `data-recording-chrome`, so
|
|
60
|
+
it cannot alter what it documents.
|
|
61
|
+
|
|
62
|
+
**Frames carry real timestamps.** The screencast delivers a frame when the compositor
|
|
63
|
+
paints one, which is irregular — a heavy Three.js scene may give 8fps for a second and
|
|
64
|
+
30fps the next. Every frame's true timestamp goes into an ffmpeg concat manifest and the
|
|
65
|
+
encode resamples to a constant rate. Treating irregular frames as evenly spaced is what
|
|
66
|
+
makes naive WebGL captures appear to speed up and slow down.
|
|
67
|
+
|
|
68
|
+
## Two capture modes
|
|
69
|
+
|
|
70
|
+
### `realtime` (default)
|
|
71
|
+
|
|
72
|
+
The compositor paints when it can and every frame carries a real timestamp. Fast to run —
|
|
73
|
+
roughly wall-clock — but the output is bound to machine speed:
|
|
74
|
+
|
|
75
|
+
| content | measured motion fps |
|
|
76
|
+
|---|---|
|
|
77
|
+
| DOM pages, scrolling and typing | 20–23 |
|
|
78
|
+
| Three.js under SwiftShader | ~10 |
|
|
79
|
+
|
|
80
|
+
No launch flag lifts this. `--disable-frame-rate-limit` and `--disable-gpu-vsync` were
|
|
81
|
+
measured and made it **worse** (15.9fps), so they are deliberately absent. Dropping JPEG
|
|
82
|
+
quality from 92 to 35 bought about 3fps, which is why the default sits at 80.
|
|
83
|
+
|
|
84
|
+
There is a second, larger problem that is easy to miss: **realtime output does not match
|
|
85
|
+
the duration the script asks for.** Driver round trips, paint waits and per-keystroke IPC
|
|
86
|
+
all add to every action, and they compound. The four-shot `studio-tour` declares about 21
|
|
87
|
+
seconds and produced **87.5 seconds** of video. A director writing a 30-second social clip
|
|
88
|
+
was getting two minutes.
|
|
89
|
+
|
|
90
|
+
### `deterministic`
|
|
91
|
+
|
|
92
|
+
Page time is replaced by a virtual clock (`clock.mjs`, installed before any page script
|
|
93
|
+
runs). The recorder advances it exactly one frame, runs the timers and animation callbacks
|
|
94
|
+
that became due, screenshots, and repeats.
|
|
95
|
+
|
|
96
|
+
| | realtime | deterministic |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| frame rate | 10–23, varies | exactly `fps` |
|
|
99
|
+
| duration vs script | ~4x over | matches |
|
|
100
|
+
| same script twice | different | **byte-identical** |
|
|
101
|
+
| wall-clock cost | ~1x | ~6x |
|
|
102
|
+
|
|
103
|
+
Measured on `studio-tour`: 515 frames at exactly 24fps, 21.5s of video from a script
|
|
104
|
+
declaring ~21s, 152ms of real work per frame, and two runs producing identical frame
|
|
105
|
+
checksums. The 3D shot has 50 unique frames out of 50 — the render loop is genuinely being
|
|
106
|
+
driven, not frozen.
|
|
107
|
+
|
|
108
|
+
**Why a JS shim rather than CDP `Emulation.setVirtualTimePolicy`:** virtual time freezes
|
|
109
|
+
the whole renderer, so a page awaiting a real network response hangs until the policy is
|
|
110
|
+
nursed through pending fetches. The shim replaces only the *timing* APIs, so network,
|
|
111
|
+
decoding and layout continue on real time and a page that fetches mid-recording simply
|
|
112
|
+
works.
|
|
113
|
+
|
|
114
|
+
**What it cannot virtualise:** `<video>`/`<audio>` playback; CSS animations are *sampled*
|
|
115
|
+
rather than stepped (paused, with `currentTime` set each frame — exact for time-based
|
|
116
|
+
animations, wrong for anything keyed to `transitionend` timing); `new Date()`, whose
|
|
117
|
+
constructor is deliberately left alone because replacing it breaks date maths in harder-to-
|
|
118
|
+
see ways; and worker or iframe timers, which have their own global scopes.
|
|
119
|
+
|
|
120
|
+
**Reproducibility has a sharp edge.** Anything non-deterministic in an action breaks it.
|
|
121
|
+
Keystroke jitter is seeded (`resetJitter`, xorshift32) and the gate fails on `Math.random`
|
|
122
|
+
or a direct `page.waitForTimeout` anywhere in `actions.mjs` — either would silently opt out
|
|
123
|
+
of the virtual clock while still looking correct in realtime mode.
|
|
124
|
+
|
|
125
|
+
### Which to use
|
|
126
|
+
|
|
127
|
+
Realtime for iterating on a script: it runs at 1x and you are looking at composition, not
|
|
128
|
+
smoothness. Deterministic for anything anyone else will see.
|
|
129
|
+
|
|
130
|
+
## The critic
|
|
131
|
+
|
|
132
|
+
Every recording is judged from its own frames and writes `critique.json` beside the video.
|
|
133
|
+
No model call — it runs on pixels and the capture timeline, so it works with no egress and
|
|
134
|
+
gives the same answer twice.
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
✓ critique 100/100 — 0 finding(s)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
| axis | weight | catches |
|
|
141
|
+
|---|---|---|
|
|
142
|
+
| `deadFrames` | 0.30 | a shot that rendered nothing — blank or a uniform wash |
|
|
143
|
+
| `frozenMotion` | 0.30 | a shot claiming movement whose frames never change |
|
|
144
|
+
| `continuity` | 0.15 | an isolated jump inside a shot, where the eye reads a cut |
|
|
145
|
+
| `pacing` | 0.15 | a shot too short for anyone to read |
|
|
146
|
+
| `ending` | 0.10 | a clip that lands mid-motion instead of on a held state |
|
|
147
|
+
|
|
148
|
+
`frozenMotion` is the one worth having. A WebGL scene that never animated produces frames
|
|
149
|
+
that each look perfectly correct, so a reviewer flicking through stills cannot see it —
|
|
150
|
+
only comparing consecutive frames can. Findings carry a timecode and the shot id, and a
|
|
151
|
+
frozen-shot finding quotes the shot's stated intent back, which is what the required
|
|
152
|
+
`intent` field is for.
|
|
153
|
+
|
|
154
|
+
Add `--gate` to make the verdict decide the exit code, for a loop that should stop on a
|
|
155
|
+
bad take rather than quietly producing one.
|
|
156
|
+
|
|
157
|
+
**The plan for this phase was wrong and was corrected.** It said to point `design-eval`'s
|
|
158
|
+
still-image rubric at the samples. That rubric's axes are `subjectFirst`, `framing`,
|
|
159
|
+
`geometryCredibility` and `materials` — product-photography measures. A frame of someone
|
|
160
|
+
filling in a form has no subject on a background and no material response, so those axes
|
|
161
|
+
would have scored noise and dressed it in a calibrated-looking number. What carried over
|
|
162
|
+
is the *shape*: measured axes, blockers separated from warnings, and a `skip` status that
|
|
163
|
+
renormalises rather than scoring a default.
|
|
164
|
+
|
|
165
|
+
### Two things it got wrong before it got them right
|
|
166
|
+
|
|
167
|
+
**Comparing luminance aggregates could not see movement.** The first version diffed
|
|
168
|
+
`imageStats`'s mean, spread and occupancy. A pure translation leaves all three unchanged,
|
|
169
|
+
so a scrolling repeated pattern — or a 3D object rotating under constant light — measured
|
|
170
|
+
as perfectly still, and the critic reported moving shots as frozen. It now compares 16x16
|
|
171
|
+
downsampled luminance grids (`frame-grid.mjs`), which see position. Coarse on purpose:
|
|
172
|
+
a pixel-exact comparison would read compression noise as constant motion.
|
|
173
|
+
|
|
174
|
+
**Large change is not the same as discontinuity.** Scrolling the landing page from its
|
|
175
|
+
dark hero into the cream section below produces enormous frame-to-frame deltas, and every
|
|
176
|
+
one was reported as a cut. A jump is *isolated* — one large change among small ones —
|
|
177
|
+
so it must also stand clear of its local median. Separately, the analysis strip is coarser
|
|
178
|
+
than a shot boundary, so the delta straddling a `goto` was being filed under the shot it
|
|
179
|
+
landed in; both ends of a shot are now treated as legitimate cuts.
|
|
180
|
+
|
|
181
|
+
Both survive as regression cases in `tools/test-record-critic.mjs`, alongside a fixture
|
|
182
|
+
for every axis. That file lives apart from `test-record.mjs` because it needs a pixel
|
|
183
|
+
backend, which CI installs only for the `tests` job — and it **fails rather than skips**
|
|
184
|
+
when the backend is missing, since a silently skipped gate is how one stops gating.
|
|
185
|
+
|
|
186
|
+
## Edits — cutting a short piece from a master
|
|
187
|
+
|
|
188
|
+
A social clip is not a different recording, it is a different *reading* of one. Cutting
|
|
189
|
+
from the master keeps the two consistent, costs seconds rather than minutes, and means the
|
|
190
|
+
30-second version cannot drift from the two-minute version it claims to summarise.
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
npm run record:edit -- recordings/studio-tour/social.edit.json --check # resolve only
|
|
194
|
+
npm run record:edit -- recordings/studio-tour/social.edit.json
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
```json
|
|
198
|
+
{
|
|
199
|
+
"name": "studio-tour-social",
|
|
200
|
+
"source": "studio-tour",
|
|
201
|
+
"aspect": "9:16",
|
|
202
|
+
"budgetSec": 30,
|
|
203
|
+
"segments": [
|
|
204
|
+
{ "shot": "start", "from": 0.6, "to": 6.5, "fit": "contain",
|
|
205
|
+
"kicker": "Brand it live", "caption": "Your palette, applied as you type." },
|
|
206
|
+
{ "shot": "webgl", "from": 2.0, "focusX": 0.62,
|
|
207
|
+
"kicker": "Real WebGL", "caption": "Live 3D — not a video of it." }
|
|
208
|
+
]
|
|
209
|
+
}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Segments address the master **by shot id**, resolved against the timeline in
|
|
213
|
+
`capture.json`, so an edit survives the script being re-timed — shots move, the edit still
|
|
214
|
+
points at the right material. `from`/`to` alongside a shot are offsets *within* it, which
|
|
215
|
+
is how a director thinks about trimming: "the last two seconds of the orbit", not "second
|
|
216
|
+
19.6 of the file". An edit naming a shot that no longer exists is refused rather than cut
|
|
217
|
+
from zero.
|
|
218
|
+
|
|
219
|
+
| field | |
|
|
220
|
+
|---|---|
|
|
221
|
+
| `aspect` | `16:9` · `9:16` · `1:1` |
|
|
222
|
+
| `fit` | `cover` crops to fill (default) · `contain` letterboxes on the studio ground |
|
|
223
|
+
| `focusX` | 0–1, where a `cover` crop sits. Per segment, falling back to the edit |
|
|
224
|
+
| `speed` | 0.25–4, retimes a segment |
|
|
225
|
+
| `budgetSec` | reported when exceeded, never enforced |
|
|
226
|
+
| `caption` · `kicker` | burned in, in the studio's own typefaces |
|
|
227
|
+
|
|
228
|
+
`budgetSec` advises rather than trimming: silently retiming a cut to hit a number changes
|
|
229
|
+
the pacing the director chose, and pacing is the whole point of a short format.
|
|
230
|
+
|
|
231
|
+
### Captions are rendered by Chromium
|
|
232
|
+
|
|
233
|
+
The obvious route is ffmpeg's `drawtext`. It is not available — the bundled `ffmpeg-static`
|
|
234
|
+
reports `--enable-libfreetype` but ships **no `drawtext` filter**, leaving only libass,
|
|
235
|
+
which renders in whatever face fontconfig finds on the machine. A caption that is one
|
|
236
|
+
typeface locally and another in CI is not a caption, it is a defect that appears only in
|
|
237
|
+
the deliverable.
|
|
238
|
+
|
|
239
|
+
So captions are rendered by the browser the recorder already launches, using the studio's
|
|
240
|
+
own `tokens.css`, and overlaid as transparent PNGs. Real typeface, real kerning, matching
|
|
241
|
+
the pages being recorded — with no font vendored into the repo.
|
|
242
|
+
|
|
243
|
+
### Framing a landscape master vertically
|
|
244
|
+
|
|
245
|
+
A 1440x900 master narrowed to 9:16 keeps only **35% of its width**. That frames a centred
|
|
246
|
+
3D stage well and slices the left column off a two-column form. Hence `fit`:
|
|
247
|
+
|
|
248
|
+
- `cover` where a *detail* is the subject — a 3D stage, a single control.
|
|
249
|
+
- `contain` where the *layout* is the subject — a form, a dense page. It letterboxes on the
|
|
250
|
+
studio background, and the resulting empty band is where the caption sits.
|
|
251
|
+
|
|
252
|
+
For a vertical piece that is mostly dense UI, the better answer is to record a vertical
|
|
253
|
+
take rather than crop a landscape one — `setViewport`, or a script with a vertical
|
|
254
|
+
`viewport`.
|
|
255
|
+
|
|
256
|
+
## Roadmap
|
|
257
|
+
|
|
258
|
+
**Phase 1 — done.** Multi-page flows with preserved state, real interactions, live page
|
|
259
|
+
manipulation, WebGL captured as painted, MP4/GIF/vertical output, a schema gate and a
|
|
260
|
+
dry run.
|
|
261
|
+
|
|
262
|
+
**Phase 2 — done.** Deterministic capture, above. Both modes share one timing seam
|
|
263
|
+
(`timeline.mjs`): actions ask the driver to `wait` or to `animate`, and the driver decides
|
|
264
|
+
whether that means sleeping or advancing a virtual frame. That is what let phase 2 land
|
|
265
|
+
without rewriting the verb vocabulary.
|
|
266
|
+
|
|
267
|
+
**Phase 3 — done.** The frame critic, below.
|
|
268
|
+
|
|
269
|
+
**Phase 4 — done.** Edits, below.
|
|
270
|
+
|
|
271
|
+
## Retired
|
|
272
|
+
|
|
273
|
+
`tools/inject-recorder-all.mjs` injected a canvas-only recorder button into the blueprint
|
|
274
|
+
editors. It emitted emoji, which `tools/verify.mjs` treats as an **error**, so running it
|
|
275
|
+
broke the build; and `canvas.captureStream()` sees only the 3D surface, so the DOM and all
|
|
276
|
+
text overlays were invisible in the result. It now exits with a pointer here.
|
|
277
|
+
|
|
278
|
+
`assets/lib/recorder.js` is still loaded by the blueprint editors and was not removed — it
|
|
279
|
+
does a genuinely different job, letting a customer export their own 3D scene from the
|
|
280
|
+
browser.
|
package/package.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@mrkt_frwd/reel",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Reel — drive a real browser from a declarative script and record it. Deterministic mode gives the same frames every run.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./src/index.mjs"
|
|
8
|
+
},
|
|
9
|
+
"bin": {
|
|
10
|
+
"reel": "./src/cli.mjs"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"src",
|
|
14
|
+
"docs"
|
|
15
|
+
],
|
|
16
|
+
"dependencies": {
|
|
17
|
+
"@playwright/browser-chromium": "^1.62.1",
|
|
18
|
+
"ffmpeg-static": "^5.3.0"
|
|
19
|
+
},
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=20"
|
|
22
|
+
},
|
|
23
|
+
"license": "MIT"
|
|
24
|
+
}
|
package/src/actions.mjs
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The action vocabulary a recording script may use.
|
|
3
|
+
*
|
|
4
|
+
* Every verb is declarative and named, because the scripts are written and critiqued by
|
|
5
|
+
* different agents: a director can say "beat 3 lingers" and an interaction developer can
|
|
6
|
+
* change one number, which is not true of a pile of imperative Playwright calls.
|
|
7
|
+
*
|
|
8
|
+
* Motion is eased rather than instant throughout. A cursor that teleports and a scroll
|
|
9
|
+
* that jumps both read as a robot driving the page; the reference formats — Linear,
|
|
10
|
+
* Stripe, Raycast — are legible precisely because movement has acceleration and the eye
|
|
11
|
+
* can follow it.
|
|
12
|
+
*
|
|
13
|
+
* No verb sleeps or eases on its own. Waiting goes through `ctx.driver.wait` and motion
|
|
14
|
+
* through `ctx.driver.animate`, so the same script runs unchanged under wall-clock capture
|
|
15
|
+
* or under the deterministic clock, where a duration means a frame count rather than
|
|
16
|
+
* elapsed time. Anything that reached for `page.waitForTimeout` directly would silently
|
|
17
|
+
* opt out of deterministic mode and desynchronise from the frames around it.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const clamp = (n, lo, hi) => Math.max(lo, Math.min(hi, n));
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Seeded jitter for keystroke cadence.
|
|
24
|
+
*
|
|
25
|
+
* Typing on a metronome does not read as typing, so the delay between keys is varied —
|
|
26
|
+
* but `Math.random()` would make every run of a script produce different frames, which
|
|
27
|
+
* silently defeats deterministic capture. A counter-seeded generator keeps the variation
|
|
28
|
+
* and keeps the reproducibility.
|
|
29
|
+
*/
|
|
30
|
+
let jitterState = 0x2f6e2b1;
|
|
31
|
+
export function resetJitter(seed = 0x2f6e2b1) { jitterState = seed >>> 0; }
|
|
32
|
+
function jitter() {
|
|
33
|
+
// xorshift32 — small, fast, and good enough for spacing keystrokes.
|
|
34
|
+
jitterState ^= jitterState << 13; jitterState >>>= 0;
|
|
35
|
+
jitterState ^= jitterState >>> 17;
|
|
36
|
+
jitterState ^= jitterState << 5; jitterState >>>= 0;
|
|
37
|
+
return jitterState / 0xffffffff;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function centerOf(page, selector) {
|
|
41
|
+
const box = await page.locator(selector).first().boundingBox();
|
|
42
|
+
if (!box) throw new Error(`selector has no box on screen: ${selector}`);
|
|
43
|
+
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Glide the real mouse. The drawn cursor follows via a `pointermove` listener installed
|
|
48
|
+
* by CURSOR_INIT, so this issues one message per step rather than three — the earlier
|
|
49
|
+
* version positioned the overlay with its own `evaluate` per frame and that IPC cost, not
|
|
50
|
+
* rendering, was what held capture near 20fps during motion.
|
|
51
|
+
*/
|
|
52
|
+
async function glide(ctx, to, ms = 520) {
|
|
53
|
+
const { page, driver } = ctx;
|
|
54
|
+
const from = await page.evaluate(() => ({ x: window.__rec?._x ?? 0, y: window.__rec?._y ?? 0 }));
|
|
55
|
+
await driver.animate(ms, async (t) => {
|
|
56
|
+
await page.mouse.move(from.x + (to.x - from.x) * t, from.y + (to.y - from.y) * t);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export const ACTIONS = {
|
|
61
|
+
async goto(ctx, a) {
|
|
62
|
+
const { page, baseUrl } = ctx;
|
|
63
|
+
const url = /^https?:/i.test(a.url) ? a.url : baseUrl + (a.url.startsWith('/') ? a.url : `/${a.url}`);
|
|
64
|
+
await page.goto(url, { waitUntil: a.waitUntil || 'load', timeout: a.timeout || 30000 });
|
|
65
|
+
// Fonts settle after load and a swap mid-shot is visible in the frames.
|
|
66
|
+
await page.evaluate(() => document.fonts?.ready).catch(() => {});
|
|
67
|
+
if (a.ms) await ctx.driver.wait(a.ms);
|
|
68
|
+
},
|
|
69
|
+
|
|
70
|
+
async waitFor({ page }, a) {
|
|
71
|
+
if (a.selector) await page.locator(a.selector).first().waitFor({ state: a.state || 'visible', timeout: a.timeout || 15000 });
|
|
72
|
+
else if (a.fn) await page.waitForFunction(a.fn, undefined, { timeout: a.timeout || 15000 });
|
|
73
|
+
else throw new Error('waitFor needs a selector or fn');
|
|
74
|
+
},
|
|
75
|
+
|
|
76
|
+
/** Dead time on purpose — a beat where the viewer reads what just happened. */
|
|
77
|
+
async hold(ctx, a) {
|
|
78
|
+
const { page } = ctx;
|
|
79
|
+
await ctx.driver.wait(a.ms ?? 700);
|
|
80
|
+
},
|
|
81
|
+
|
|
82
|
+
async moveTo(ctx, a) {
|
|
83
|
+
const { page } = ctx;
|
|
84
|
+
await glide(ctx, { x: a.x, y: a.y }, a.ms);
|
|
85
|
+
},
|
|
86
|
+
|
|
87
|
+
async hover(ctx, a) {
|
|
88
|
+
const { page } = ctx;
|
|
89
|
+
await glide(ctx, await centerOf(page, a.selector), a.ms);
|
|
90
|
+
await page.locator(a.selector).first().hover({ force: true }).catch(() => {});
|
|
91
|
+
if (a.hold) await ctx.driver.wait(a.hold);
|
|
92
|
+
},
|
|
93
|
+
|
|
94
|
+
async click(ctx, a) {
|
|
95
|
+
const { page } = ctx;
|
|
96
|
+
const point = await centerOf(page, a.selector);
|
|
97
|
+
await glide(ctx, point, a.ms);
|
|
98
|
+
await page.evaluate(() => window.__rec?.ripple());
|
|
99
|
+
await page.mouse.click(point.x, point.y);
|
|
100
|
+
if (a.hold) await ctx.driver.wait(a.hold);
|
|
101
|
+
},
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Types at a human cadence rather than pasting. `cps` is characters per second; the
|
|
105
|
+
* per-key jitter keeps it off a metronome, which is what makes typing read as typing.
|
|
106
|
+
*/
|
|
107
|
+
async type(ctx, a) {
|
|
108
|
+
const { page } = ctx;
|
|
109
|
+
const el = page.locator(a.selector).first();
|
|
110
|
+
if (a.click !== false) {
|
|
111
|
+
const point = await centerOf(page, a.selector);
|
|
112
|
+
await glide(ctx, point, a.ms);
|
|
113
|
+
await page.evaluate(() => window.__rec?.ripple());
|
|
114
|
+
await page.mouse.click(point.x, point.y);
|
|
115
|
+
}
|
|
116
|
+
if (a.clear) await el.fill('');
|
|
117
|
+
const cps = a.cps || 18;
|
|
118
|
+
for (const ch of String(a.text)) {
|
|
119
|
+
await el.press(ch === ' ' ? 'Space' : ch, { timeout: 5000 }).catch(async () => {
|
|
120
|
+
await page.keyboard.type(ch);
|
|
121
|
+
});
|
|
122
|
+
await ctx.driver.wait(clamp((1000 / cps) * (0.6 + jitter() * 0.8), 12, 220));
|
|
123
|
+
}
|
|
124
|
+
if (a.hold) await ctx.driver.wait(a.hold);
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
async press(ctx, a) {
|
|
128
|
+
const { page } = ctx;
|
|
129
|
+
for (let i = 0; i < (a.times || 1); i++) {
|
|
130
|
+
await page.keyboard.press(a.key);
|
|
131
|
+
await ctx.driver.wait(a.every ?? 120);
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
|
|
135
|
+
/** Eased programmatic scroll. The page's own smooth-scroll is disabled by POLISH_INIT. */
|
|
136
|
+
async scroll(ctx, a) {
|
|
137
|
+
const { page } = ctx;
|
|
138
|
+
const target = await page.evaluate((spec) => {
|
|
139
|
+
if (typeof spec === 'number') return spec;
|
|
140
|
+
if (spec === 'bottom') return document.documentElement.scrollHeight - window.innerHeight;
|
|
141
|
+
if (spec === 'top') return 0;
|
|
142
|
+
const el = document.querySelector(spec);
|
|
143
|
+
if (!el) throw new Error('scroll target not found: ' + spec);
|
|
144
|
+
return window.scrollY + el.getBoundingClientRect().top - window.innerHeight * 0.2;
|
|
145
|
+
}, a.to);
|
|
146
|
+
|
|
147
|
+
// Stepped from the driver rather than animated in-page. An in-page requestAnimationFrame
|
|
148
|
+
// loop cannot work under the deterministic clock: rAF only fires when the driver ticks,
|
|
149
|
+
// and the driver is blocked awaiting the very promise those ticks would resolve. Driving
|
|
150
|
+
// it from outside is the only formulation that holds in both modes.
|
|
151
|
+
const from = await page.evaluate(() => window.scrollY);
|
|
152
|
+
await ctx.driver.animate(a.ms ?? 1400, async (t) => {
|
|
153
|
+
await page.evaluate((y) => window.scrollTo(0, y), from + (target - from) * t);
|
|
154
|
+
});
|
|
155
|
+
if (a.hold) await ctx.driver.wait(a.hold);
|
|
156
|
+
},
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Press, glide, release. This is how a 3D scene gets orbited — the WebGL templates
|
|
160
|
+
* read pointer deltas, so the drag has to be many small moves rather than one jump.
|
|
161
|
+
*/
|
|
162
|
+
async drag(ctx, a) {
|
|
163
|
+
const { page } = ctx;
|
|
164
|
+
const start = a.selector ? await centerOf(page, a.selector) : { x: a.x, y: a.y };
|
|
165
|
+
await glide(ctx, start, a.approachMs ?? 420);
|
|
166
|
+
await page.mouse.down();
|
|
167
|
+
await ctx.driver.animate(a.ms ?? 1200, async (t) => {
|
|
168
|
+
await page.mouse.move(start.x + (a.dx || 0) * t, start.y + (a.dy || 0) * t);
|
|
169
|
+
});
|
|
170
|
+
await page.mouse.up();
|
|
171
|
+
if (a.hold) await ctx.driver.wait(a.hold);
|
|
172
|
+
},
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Run script in the page — the lever for making the page itself dynamic mid-recording:
|
|
176
|
+
* seed data, trigger an animation, force a state the UI cannot reach quickly. The
|
|
177
|
+
* source is a local script file the operator wrote, same trust level as the tool.
|
|
178
|
+
*/
|
|
179
|
+
async eval(ctx, a) {
|
|
180
|
+
const { page } = ctx;
|
|
181
|
+
await page.evaluate(a.fn, a.arg);
|
|
182
|
+
if (a.ms) await ctx.driver.wait(a.ms);
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
async inject(ctx, a) {
|
|
186
|
+
const { page } = ctx;
|
|
187
|
+
if (a.css) await page.addStyleTag({ content: a.css });
|
|
188
|
+
if (a.js) await page.addScriptTag({ content: a.js });
|
|
189
|
+
if (a.ms) await ctx.driver.wait(a.ms);
|
|
190
|
+
},
|
|
191
|
+
|
|
192
|
+
/** Stub a network response so a demo never depends on a live service being up. */
|
|
193
|
+
async route({ page }, a) {
|
|
194
|
+
await page.route(a.url, (route) =>
|
|
195
|
+
route.fulfill({
|
|
196
|
+
status: a.status || 200,
|
|
197
|
+
contentType: a.json ? 'application/json' : a.contentType || 'text/plain',
|
|
198
|
+
body: a.json ? JSON.stringify(a.json) : (a.body ?? ''),
|
|
199
|
+
})
|
|
200
|
+
);
|
|
201
|
+
},
|
|
202
|
+
|
|
203
|
+
async setViewport(ctx, a) {
|
|
204
|
+
const { page } = ctx;
|
|
205
|
+
await page.setViewportSize({ width: a.width, height: a.height });
|
|
206
|
+
if (a.ms) await ctx.driver.wait(a.ms);
|
|
207
|
+
},
|
|
208
|
+
|
|
209
|
+
async cursor({ page }, a) {
|
|
210
|
+
if (a.hide) await page.evaluate(() => window.__rec?.hide());
|
|
211
|
+
else await page.evaluate(([x, y]) => window.__rec?.cursor(x, y), [a.x ?? 0, a.y ?? 0]);
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
export const ACTION_TYPES = Object.keys(ACTIONS);
|