@open-take/compositor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/chunk-Cl8Af3a2.js +11 -0
- package/dist/index.d.ts +482 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +973 -0
- package/dist/index.js.map +1 -0
- package/package.json +61 -0
- package/src/ffmpeg.ts +60 -0
- package/src/index.ts +21 -0
- package/src/math.ts +459 -0
- package/src/plan.ts +157 -0
- package/src/presets.ts +147 -0
- package/src/render.ts +237 -0
- package/src/scene/project.ts +19 -0
- package/src/scene/scene.tsx +231 -0
- package/src/types.ts +349 -0
- package/src/validate.ts +245 -0
package/src/types.ts
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
// The editable take composition — the source of truth. The agent (or a
|
|
2
|
+
// human) edits THIS, and the revideo scene renders it deterministically.
|
|
3
|
+
// All spatial fields are in VIDEO-pixel space (capture coords mapped
|
|
4
|
+
// through viewport→video scaling), so the scene works in one coordinate
|
|
5
|
+
// system.
|
|
6
|
+
|
|
7
|
+
export type Pt = { x: number; y: number };
|
|
8
|
+
export type BBox = { x: number; y: number; w: number; h: number };
|
|
9
|
+
|
|
10
|
+
// --- capture input (the ground-truth event log) -----------------------
|
|
11
|
+
|
|
12
|
+
/** Editorial zoom intent for an action (set by the planner/agent). */
|
|
13
|
+
export type ZoomIntent = "auto" | "never" | "always";
|
|
14
|
+
|
|
15
|
+
/** Fields common to every captured action. `x,y` is the anchor / start
|
|
16
|
+
* point (cursor target), viewport CSS px. */
|
|
17
|
+
export type CaptureEventBase = {
|
|
18
|
+
/** anchor point (click target / field / drag start), viewport CSS px */
|
|
19
|
+
x: number;
|
|
20
|
+
y: number;
|
|
21
|
+
/** element bounding box, viewport CSS px — the ground-truth edge */
|
|
22
|
+
box?: BBox;
|
|
23
|
+
/** ms from recording start (when the cursor arrives / action begins) */
|
|
24
|
+
tMs: number;
|
|
25
|
+
/** selector / note, kept for editability */
|
|
26
|
+
sel?: string;
|
|
27
|
+
note?: string;
|
|
28
|
+
/** selective-zoom intent from the plan (default auto = heuristic) */
|
|
29
|
+
zoom?: ZoomIntent;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** A click (or a type's focus-click): an instantaneous action at a point. */
|
|
33
|
+
export type CaptureClick = CaptureEventBase & { kind?: "click" };
|
|
34
|
+
|
|
35
|
+
/** Typing into a focused field: the cursor parks and the zoom holds for
|
|
36
|
+
* `durationMs` while the text appears in the recording. */
|
|
37
|
+
export type CaptureType = CaptureEventBase & {
|
|
38
|
+
kind: "type";
|
|
39
|
+
/** what was typed (editability) */
|
|
40
|
+
text: string;
|
|
41
|
+
/** ms the typing occupies on screen (ground-truth wall time) */
|
|
42
|
+
durationMs: number;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** A drag: a path from the anchor (`x,y`) to `to`, optionally via `path`,
|
|
46
|
+
* with the button held for `durationMs`. */
|
|
47
|
+
export type CaptureDrag = CaptureEventBase & {
|
|
48
|
+
kind: "drag";
|
|
49
|
+
/** drag end point, viewport CSS px */
|
|
50
|
+
to: { x: number; y: number };
|
|
51
|
+
/** full polyline incl. ends, viewport CSS px (freehand strokes) */
|
|
52
|
+
path?: { x: number; y: number }[];
|
|
53
|
+
/** ms the drag occupies on screen (ground-truth wall time) */
|
|
54
|
+
durationMs: number;
|
|
55
|
+
/** how the stroke was paced and BAKED into the ink: "smooth" (accel-in /
|
|
56
|
+
* decel-out — a natural hand-draw) or "linear" (constant speed). The
|
|
57
|
+
* compositor cursor must replay the SAME easing to stay locked to the ink.
|
|
58
|
+
* Absent ⇒ linear (legacy captures). */
|
|
59
|
+
ease?: "linear" | "smooth";
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** A scroll: the content pans for `durationMs`. The cursor holds (no travel),
|
|
63
|
+
* full-view (no zoom). `dy` is the signed pixels scrolled (editability). */
|
|
64
|
+
export type CaptureScroll = CaptureEventBase & {
|
|
65
|
+
kind: "scroll";
|
|
66
|
+
/** signed pixels scrolled (positive = down) */
|
|
67
|
+
dy: number;
|
|
68
|
+
/** ms the scroll occupies on screen (ground-truth wall time) */
|
|
69
|
+
durationMs: number;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** A hover: the cursor travels to `x,y` and dwells for `durationMs` so a
|
|
73
|
+
* tooltip / menu / hover-state reveals. Like a click that doesn't click. */
|
|
74
|
+
export type CaptureHover = CaptureEventBase & {
|
|
75
|
+
kind: "hover";
|
|
76
|
+
/** ms the dwell occupies on screen (ground-truth wall time) */
|
|
77
|
+
durationMs: number;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** A key press / shortcut: keyboard-driven, so the cursor holds (no travel).
|
|
81
|
+
* Holds for `durationMs` while the effect plays out; if a reveal element was
|
|
82
|
+
* located, `box` carries its bbox so the zoom can frame it. */
|
|
83
|
+
export type CapturePress = CaptureEventBase & {
|
|
84
|
+
kind: "press";
|
|
85
|
+
/** the chord pressed, e.g. "Enter" / "Meta+k" (editability) */
|
|
86
|
+
keys: string;
|
|
87
|
+
/** ms the hold occupies on screen (ground-truth wall time) */
|
|
88
|
+
durationMs: number;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
export type CaptureEvent =
|
|
92
|
+
| CaptureClick
|
|
93
|
+
| CaptureType
|
|
94
|
+
| CaptureDrag
|
|
95
|
+
| CaptureScroll
|
|
96
|
+
| CaptureHover
|
|
97
|
+
| CapturePress;
|
|
98
|
+
|
|
99
|
+
export type CaptureLog = {
|
|
100
|
+
video: { width: number; height: number; fps?: number | string; durationS?: number };
|
|
101
|
+
viewport: { w: number; h: number };
|
|
102
|
+
start?: { x: number; y: number };
|
|
103
|
+
/** the ordered ground-truth actions (click / type / drag) */
|
|
104
|
+
events: CaptureEvent[];
|
|
105
|
+
tEndMs?: number;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
// --- the composition (editable) ----------------------------------------
|
|
109
|
+
|
|
110
|
+
export type ZoomDecision = {
|
|
111
|
+
/** selective: not every action zooms. Edit this to tune/remove. */
|
|
112
|
+
enabled: boolean;
|
|
113
|
+
/** absolute stage scale to reach (bbox-fit, capped) */
|
|
114
|
+
scale: number;
|
|
115
|
+
/** video-px point to frame (bbox center), pre-clamp */
|
|
116
|
+
center: Pt;
|
|
117
|
+
/** when the zoom-in begins (ms) */
|
|
118
|
+
inAtMs: number;
|
|
119
|
+
/** Optional "glide": a slow camera drift WHILE the zoom is held, as a velocity
|
|
120
|
+
* in video-px per second {x,y}. The held centre pans from `center` by
|
|
121
|
+
* `glide · holdSeconds` across the hold window (then the next beat ramps from
|
|
122
|
+
* there / it zooms out from there). Adds life vs a dead-static hold (Screen
|
|
123
|
+
* Studio's glide). Absent/0 ⇒ a still hold. Clamped to the video at read time,
|
|
124
|
+
* so a drift just stops at the edge. Keep it gentle (tens of px/s). */
|
|
125
|
+
glide?: Pt;
|
|
126
|
+
/** why this decision (for the human/agent reading the composition) */
|
|
127
|
+
reason: string;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
export type CompEvent = {
|
|
131
|
+
kind: "click" | "type" | "drag" | "scroll" | "hover" | "press";
|
|
132
|
+
tMs: number;
|
|
133
|
+
/** anchor point (click / focus / drag start / hover) in video-px. For a
|
|
134
|
+
* scroll/press the cursor does not move; this is its resting point. */
|
|
135
|
+
point: Pt;
|
|
136
|
+
/** element bbox in video-px (if known) */
|
|
137
|
+
bbox?: BBox;
|
|
138
|
+
label?: string;
|
|
139
|
+
zoom: ZoomDecision;
|
|
140
|
+
/** how long the action plays out after `tMs` (type/drag/scroll/hover/press);
|
|
141
|
+
* 0 for a click. The cursor parks and the zoom holds for this long. */
|
|
142
|
+
durationMs?: number;
|
|
143
|
+
/** typed text (kind=type), for editability */
|
|
144
|
+
text?: string;
|
|
145
|
+
/** chord pressed (kind=press), for editability */
|
|
146
|
+
keys?: string;
|
|
147
|
+
/** drag end point, video-px (kind=drag) */
|
|
148
|
+
to?: Pt;
|
|
149
|
+
/** drag polyline incl. ends, video-px (kind=drag) — the cursor path */
|
|
150
|
+
path?: Pt[];
|
|
151
|
+
/** drag stroke easing baked into the ink (kind=drag): "smooth" or "linear".
|
|
152
|
+
* The cursor replays it so it stays locked to the ink. Absent ⇒ linear. */
|
|
153
|
+
ease?: "linear" | "smooth";
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
export type FramingConfig = {
|
|
157
|
+
/** video occupies this fraction of the frame at rest (inset for the backdrop) */
|
|
158
|
+
insetFrac: number;
|
|
159
|
+
cornerRadius: number;
|
|
160
|
+
shadow: { color: string; blur: number; offset: Pt };
|
|
161
|
+
/** backdrop behind the framed video. `type` defaults to "gradient" (from→to);
|
|
162
|
+
* "solid" fills `from` only. `angle` (deg, CSS-like: 0 = upward) rotates the
|
|
163
|
+
* gradient; absent ⇒ the legacy top-left→bottom-right diagonal (pixel-identical). */
|
|
164
|
+
background: { from: string; to: string; type?: "gradient" | "solid"; angle?: number };
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
export type CursorConfig = {
|
|
168
|
+
/** Fallback travel duration (ms) when `travelWidthsPerSec` is unset/0. A FIXED
|
|
169
|
+
* duration makes cursor speed scale with distance (short=slow, long=fast),
|
|
170
|
+
* which reads inconsistent; prefer the distance-aware speed below. */
|
|
171
|
+
travelMs: number;
|
|
172
|
+
/** Distance-aware travel speed, as a fraction of the source video WIDTH per
|
|
173
|
+
* second (resolution-independent). A travel leg's duration is
|
|
174
|
+
* `clamp(distance / (widthsPerSec·videoWidth), travelMinMs, travelMaxMs)`, so
|
|
175
|
+
* the cursor holds a roughly CONSTANT on-screen speed regardless of distance —
|
|
176
|
+
* the premium-recorder feel (measured ~0.30 widths/s on the reference). Set 0
|
|
177
|
+
* to fall back to the fixed `travelMs`. */
|
|
178
|
+
travelWidthsPerSec: number;
|
|
179
|
+
/** Floor for a distance-aware travel (ms) so short hops aren't an instant snap. */
|
|
180
|
+
travelMinMs: number;
|
|
181
|
+
/** Ceiling for a distance-aware travel (ms) so a full-width jump stays a glide,
|
|
182
|
+
* not a multi-second crawl. */
|
|
183
|
+
travelMaxMs: number;
|
|
184
|
+
scale: number;
|
|
185
|
+
arcFrac: number;
|
|
186
|
+
arcMax: number;
|
|
187
|
+
rippleMs: number;
|
|
188
|
+
/** ms to hold a zoom after the action settles, before zooming back out */
|
|
189
|
+
holdMs: number;
|
|
190
|
+
/** ms for the zoom-OUT ramp (back to rest) */
|
|
191
|
+
zoomOutMs: number;
|
|
192
|
+
/** ms for the zoom-IN ramp (into a target). Decoupled from travelMs so the
|
|
193
|
+
* zoom can be slower/gentler than the cursor (a cinematic ~1s zoom). */
|
|
194
|
+
zoomInMs: number;
|
|
195
|
+
/** Easing for the zoom/pan stage ramps (scale + center together), as
|
|
196
|
+
* cubic-bezier control points. Absent ⇒ symmetric smootherstep — whose broad
|
|
197
|
+
* near-constant-velocity middle reads a bit linear, esp. on the zoom-OUT
|
|
198
|
+
* settle. A decel-biased curve gives a softer landing into rest. */
|
|
199
|
+
zoomEase?: [number, number, number, number];
|
|
200
|
+
/** Spring easing for the zoom/pan stage ramps, as a `bounce` amount ∈ [0,~0.6):
|
|
201
|
+
* 0 = critically damped (a soft physical ease-out), higher = more overshoot/
|
|
202
|
+
* snap (the "silky" settle a premium screen-recorder has; ~0.06 for zoom). When
|
|
203
|
+
* set, this WINS over `zoomEase` (see math.ts stageEasing). Absent ⇒ use
|
|
204
|
+
* `zoomEase`/smootherstep. The segment duration stays zoomInMs/zoomOutMs;
|
|
205
|
+
* bounce only shapes the curve. NB: large bounce can undershoot rest on the
|
|
206
|
+
* zoom-OUT (momentary backdrop dead-space) — keep it small for zoom. */
|
|
207
|
+
zoomSpring?: number;
|
|
208
|
+
/** ms to delay the synthetic cursor along a DRAG stroke, compensating for the
|
|
209
|
+
* capture pipeline latency: the captured ink appears ~this long after the pen
|
|
210
|
+
* actually moved, so without the delay the (exact-time) cursor leads the ink.
|
|
211
|
+
* Tune so the cursor tip sits on the ink front mid-stroke. */
|
|
212
|
+
dragLagMs: number;
|
|
213
|
+
/** easing for a travel move, as cubic-bezier control points [x1,y1,x2,y2].
|
|
214
|
+
* Default is a symmetric ease-in-out — measured from a reference recording, whose
|
|
215
|
+
* cursor accelerates and decelerates evenly (a slow start, fast middle, soft
|
|
216
|
+
* landing). Drag strokes ignore this (they ease the stroke in lockstep with
|
|
217
|
+
* the captured ink — see math.ts). */
|
|
218
|
+
travelEase: [number, number, number, number];
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
/** Camera motion blur (temporal supersampling — what a real shutter does). The
|
|
222
|
+
* renderer samples the composition camera at `samples` sub-times within each
|
|
223
|
+
* output frame and averages them, so a fast zoom/pan/cursor smears in the
|
|
224
|
+
* motion direction (a camera/shutter motion blur). It smears the
|
|
225
|
+
* backdrop-reveal on a zoom-OUT into a soft gradient instead of a hard
|
|
226
|
+
* single-frame pop. The captured video's frames repeat across sub-samples, so
|
|
227
|
+
* the recording's own content is NOT blurred — only the camera move + cursor. */
|
|
228
|
+
export type MotionBlurConfig = {
|
|
229
|
+
/** sub-frames sampled per output frame. 1 ⇒ OFF (no supersampling, no cost). */
|
|
230
|
+
samples: number;
|
|
231
|
+
/** fraction of the frame interval the shutter is open (0..1). Blur strength;
|
|
232
|
+
* 0 ⇒ OFF. ~0.5 = a 180° shutter, 1 = 360°. */
|
|
233
|
+
shutter: number;
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/** One caption window on a review copy: the pill reads `text` from `fromMs`
|
|
237
|
+
* (inclusive) to `toMs` (exclusive) on the composition timeline. */
|
|
238
|
+
export type ReviewBadge = { fromMs: number; toMs: number; text: string };
|
|
239
|
+
|
|
240
|
+
/** Render-time decoration for review copies and A/B variant reels: burned-in
|
|
241
|
+
* beat badges, a REVIEW watermark, and a per-variant corner label. Drawn by the
|
|
242
|
+
* scene in SCREEN space (fixed, outside the composition camera) so the text is
|
|
243
|
+
* legible at any zoom. This is a render input only — it is never written into
|
|
244
|
+
* the editable `*.composition.json` artifact (render strips it) and the
|
|
245
|
+
* validator ignores it. Text renders in the headless Chrome, so any script
|
|
246
|
+
* (incl. CJK) works without ffmpeg font filters. */
|
|
247
|
+
export type ReviewDecor = {
|
|
248
|
+
/** top-right watermark, e.g. "REVIEW" — marks a copy as not-for-posting */
|
|
249
|
+
watermark?: string;
|
|
250
|
+
/** bottom-left caption pill, swapping per timeline window */
|
|
251
|
+
badges?: ReviewBadge[];
|
|
252
|
+
/** constant bottom-left variant label for A/B reels, e.g. "B · tight ×1.8" */
|
|
253
|
+
label?: string;
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
export type TakeComposition = {
|
|
257
|
+
output: { width: number; height: number; fps: number };
|
|
258
|
+
source: {
|
|
259
|
+
videoUrl: string;
|
|
260
|
+
videoWidth: number;
|
|
261
|
+
videoHeight: number;
|
|
262
|
+
viewport: { w: number; h: number };
|
|
263
|
+
};
|
|
264
|
+
framing: FramingConfig;
|
|
265
|
+
cursor: CursorConfig;
|
|
266
|
+
/** camera motion blur; absent ⇒ off (renders exactly as before). */
|
|
267
|
+
motionBlur?: MotionBlurConfig;
|
|
268
|
+
/** cursor start, video-px */
|
|
269
|
+
start: Pt;
|
|
270
|
+
events: CompEvent[];
|
|
271
|
+
durationMs: number;
|
|
272
|
+
/** render-time review decoration (badges/watermark/label); never persisted */
|
|
273
|
+
review?: ReviewDecor;
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
/** True when motion blur is configured to actually do something (so the OFF
|
|
277
|
+
* path stays byte-identical to the pre-motion-blur renderer). */
|
|
278
|
+
export function motionBlurActive(mb: MotionBlurConfig | undefined): mb is MotionBlurConfig {
|
|
279
|
+
return !!mb && mb.samples > 1 && mb.shutter > 0;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export const DEFAULT_FRAMING: FramingConfig = {
|
|
283
|
+
insetFrac: 0.92,
|
|
284
|
+
cornerRadius: 28,
|
|
285
|
+
shadow: { color: "rgba(0,0,0,0.55)", blur: 60, offset: { x: 0, y: 28 } },
|
|
286
|
+
background: { from: "#1e1b3a", to: "#0a0e1c" },
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
// Camera motion blur, ON by default — it smooths the zoom
|
|
290
|
+
// motion and softens the zoom-OUT backdrop reveal (the model-C hitch). 6
|
|
291
|
+
// sub-frames is a good smoothness/speed balance; ~0.7 shutter is a strong-ish
|
|
292
|
+
// blur without ghosting. EXPORT render cost scales with `samples` (renders at
|
|
293
|
+
// fps·samples then averages), so `samples` is the quality⇄speed knob — drop it
|
|
294
|
+
// to render faster, raise (≤~12) for silkier fast pans.
|
|
295
|
+
export const DEFAULT_MOTION_BLUR: MotionBlurConfig = {
|
|
296
|
+
samples: 6,
|
|
297
|
+
shutter: 0.7,
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
export const DEFAULT_CURSOR: CursorConfig = {
|
|
301
|
+
travelMs: 560,
|
|
302
|
+
// Distance-aware cursor speed — the dominant "silky" lever. A fixed duration
|
|
303
|
+
// makes velocity scale with distance (short=slow, long=fast); holding a roughly
|
|
304
|
+
// constant on-screen speed reads consistent/premium. ~0.35 widths/s with the
|
|
305
|
+
// big-move cap (travelMaxMs 850) was tuned by eye to a notch slower than a
|
|
306
|
+
// premium reference recording (svgl.mp4) on this app's long toolbar→canvas
|
|
307
|
+
// moves. Tune up for snappier, down for slower/grander.
|
|
308
|
+
// NOTE on the cap (subtle): travel ENDS at the action's instant, so a LONGER
|
|
309
|
+
// duration starts EARLIER, not later. With the front-loaded ease (below) a
|
|
310
|
+
// longer move therefore reaches the target SOONER — i.e. raising travelMaxMs
|
|
311
|
+
// can read as *faster*. Judge speed by how fast the cursor SWEEPS, not by when
|
|
312
|
+
// it arrives; lower the cap for a more deliberate sweep.
|
|
313
|
+
travelWidthsPerSec: 0.35,
|
|
314
|
+
travelMinMs: 300,
|
|
315
|
+
travelMaxMs: 850,
|
|
316
|
+
scale: 2.0,
|
|
317
|
+
// Near-straight glide. The reference cursor barely bows (measured: a
|
|
318
|
+
// full-width move deviated <2% off the straight line), so the arc is small —
|
|
319
|
+
// just enough to avoid a robotic ruler-straight path, not a visible curve.
|
|
320
|
+
arcFrac: 0.05,
|
|
321
|
+
arcMax: 24,
|
|
322
|
+
rippleMs: 450,
|
|
323
|
+
holdMs: 1100,
|
|
324
|
+
// Gentle, cinematic zoom (a ~1s ramp reads as premium); our
|
|
325
|
+
// old 600ms tied-to-travel zoom felt snappy/mechanical by comparison.
|
|
326
|
+
zoomOutMs: 800,
|
|
327
|
+
zoomInMs: 760,
|
|
328
|
+
// Zoom/pan stage easing. Same decel-biased curve as the cursor — symmetric
|
|
329
|
+
// smootherstep (the fallback) has a broad near-constant-velocity middle that
|
|
330
|
+
// reads a touch linear, especially as the zoom-OUT settles back to rest; this
|
|
331
|
+
// gives a soft landing into rest. Applied to scale + center together.
|
|
332
|
+
zoomEase: [0.3, 0.0, 0.2, 1.0],
|
|
333
|
+
// The captured ink trails the pen by the screencast/encode pipeline latency τ;
|
|
334
|
+
// delay the cursor by τ so its tip rides the ink front. Set to τ EXACTLY and
|
|
335
|
+
// the cursor locks to the ink at ALL stroke speeds (both are the same time-
|
|
336
|
+
// delay of the same path), which is what makes a SMOOTH (eased) drag work —
|
|
337
|
+
// its fast mid-section amplifies any τ mismatch into a visible lead. Measured
|
|
338
|
+
// τ≈190ms on this pipeline (swept dragLagMs, picked where the tip sits on the
|
|
339
|
+
// ink front mid-stroke); the old 110ms left even linear strokes ~40px ahead.
|
|
340
|
+
// Re-tune if a different machine's encode latency differs.
|
|
341
|
+
dragLagMs: 190,
|
|
342
|
+
// Soft launch + soft landing (gentle accel from rest, peak ~t0.24, decelerate
|
|
343
|
+
// into the target). Chosen over a symmetric ease-in-out (which felt less silky)
|
|
344
|
+
// and over a pure ease-out [.33,1,.68,1] (instant-max launch + a 46%-of-duration
|
|
345
|
+
// creep tail that makes the speed knob lie — see travelMaxMs). This keeps the
|
|
346
|
+
// soft landing (the silk) while trimming the tail to 38% and removing the abrupt
|
|
347
|
+
// launch. Tune toward [0.42,0,0.58,1] for a more uniform sweep.
|
|
348
|
+
travelEase: [0.3, 0.0, 0.2, 1.0],
|
|
349
|
+
};
|
package/src/validate.ts
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// validateComposition: a NON-MUTATING structural check on an edited
|
|
2
|
+
// composition. The refine loop is "edit the JSON, re-render" — so the agent
|
|
3
|
+
// (or a human) hand-edits `*.composition.json`, and this gives a
|
|
4
|
+
// millisecond, field-specific verdict BEFORE paying for a render. It never
|
|
5
|
+
// changes the composition; it only reports.
|
|
6
|
+
//
|
|
7
|
+
// Severity:
|
|
8
|
+
// "error" — the render would be wrong/broken (or silently mis-framed).
|
|
9
|
+
// renderTake refuses to render until these are fixed.
|
|
10
|
+
// "warn" — suspect, but may be intentional. Rendered as-is.
|
|
11
|
+
//
|
|
12
|
+
// The single most important check is the capture-lock (see CAPTURE-LOCKED
|
|
13
|
+
// below): an action beat's `tMs` is bound to the recording — the video frame
|
|
14
|
+
// at time t shows the page AS IT WAS at capture-time t — so re-rendering with
|
|
15
|
+
// a moved `tMs` desyncs the overlay from the on-screen action. Retiming an
|
|
16
|
+
// action needs a re-capture, not a JSON edit. Pass the capture log to enforce
|
|
17
|
+
// this; without it the check is skipped (we can't know the ground truth).
|
|
18
|
+
|
|
19
|
+
import { restStageScale } from "./math";
|
|
20
|
+
import type { CaptureLog, TakeComposition } from "./types";
|
|
21
|
+
|
|
22
|
+
export type CompositionIssue = {
|
|
23
|
+
severity: "error" | "warn";
|
|
24
|
+
/** dotted/indexed field path, e.g. "events[3].zoom.scale" */
|
|
25
|
+
path: string;
|
|
26
|
+
message: string;
|
|
27
|
+
/** a concrete suggested correction the agent can apply */
|
|
28
|
+
fix?: string;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export type ValidateOpts = {
|
|
32
|
+
/** soft ceiling on zoom scale — above it the pixels visibly soften (warn).
|
|
33
|
+
* Default 2.5 (the planner caps auto-zoom at 1.5; manual edits get headroom). */
|
|
34
|
+
maxScale?: number;
|
|
35
|
+
/** the ground-truth capture log. When given, action `tMs` is checked against
|
|
36
|
+
* it (capture-lock). Omit to skip that check. */
|
|
37
|
+
captureLog?: CaptureLog;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const reqField: Record<string, string> = {
|
|
41
|
+
type: "text",
|
|
42
|
+
press: "keys",
|
|
43
|
+
drag: "to",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
export function validateComposition(
|
|
47
|
+
comp: TakeComposition,
|
|
48
|
+
opts: ValidateOpts = {},
|
|
49
|
+
): CompositionIssue[] {
|
|
50
|
+
const issues: CompositionIssue[] = [];
|
|
51
|
+
const err = (path: string, message: string, fix?: string) =>
|
|
52
|
+
issues.push({ severity: "error", path, message, fix });
|
|
53
|
+
const warn = (path: string, message: string, fix?: string) =>
|
|
54
|
+
issues.push({ severity: "warn", path, message, fix });
|
|
55
|
+
|
|
56
|
+
const maxScale = opts.maxScale ?? 2.5;
|
|
57
|
+
|
|
58
|
+
// --- output / source sanity ---
|
|
59
|
+
const { width: oW, height: oH, fps } = comp.output;
|
|
60
|
+
if (!(oW > 0 && oH > 0))
|
|
61
|
+
err("output", `non-positive output size ${oW}x${oH}`, "set positive width/height");
|
|
62
|
+
if (!(fps > 0)) err("output.fps", `fps must be > 0 (got ${fps})`, "set output.fps to 30 or 60");
|
|
63
|
+
const { videoWidth: vW, videoHeight: vH } = comp.source;
|
|
64
|
+
if (!(vW > 0 && vH > 0)) err("source", `non-positive source video size ${vW}x${vH}`);
|
|
65
|
+
|
|
66
|
+
const rest = restStageScale(vW, vH, oW, oH, comp.framing.insetFrac);
|
|
67
|
+
|
|
68
|
+
// --- motion blur (render cost scales with samples: frames = fps × samples) ---
|
|
69
|
+
const mb = comp.motionBlur;
|
|
70
|
+
if (mb) {
|
|
71
|
+
if (!Number.isInteger(mb.samples) || mb.samples < 1 || mb.samples > 16)
|
|
72
|
+
err(
|
|
73
|
+
"motionBlur.samples",
|
|
74
|
+
`samples must be an integer 1-16 (got ${mb.samples})`,
|
|
75
|
+
"6 is the balanced default; 1 turns blur off",
|
|
76
|
+
);
|
|
77
|
+
else if (mb.samples > 9)
|
|
78
|
+
warn(
|
|
79
|
+
"motionBlur.samples",
|
|
80
|
+
`samples ${mb.samples} renders ${mb.samples}× the frames for little extra smoothness`,
|
|
81
|
+
"9 is the practical ceiling",
|
|
82
|
+
);
|
|
83
|
+
if (!(mb.shutter >= 0 && mb.shutter <= 1))
|
|
84
|
+
err(
|
|
85
|
+
"motionBlur.shutter",
|
|
86
|
+
`shutter must be 0..1 (got ${mb.shutter})`,
|
|
87
|
+
"0.7 is the default; 0 turns blur off",
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// --- duration / ordering ---
|
|
92
|
+
const events = comp.events ?? [];
|
|
93
|
+
let lastEnd = 0;
|
|
94
|
+
let prevT = -1;
|
|
95
|
+
for (let i = 0; i < events.length; i++) {
|
|
96
|
+
const e = events[i]!;
|
|
97
|
+
const p = `events[${i}]`;
|
|
98
|
+
const dur = e.durationMs ?? 0;
|
|
99
|
+
const endMs = e.tMs + dur;
|
|
100
|
+
lastEnd = Math.max(lastEnd, endMs);
|
|
101
|
+
|
|
102
|
+
// ordering: events must stay in temporal order (the video is temporal)
|
|
103
|
+
if (e.tMs < prevT) {
|
|
104
|
+
err(
|
|
105
|
+
`${p}.tMs`,
|
|
106
|
+
`tMs ${e.tMs} is before the previous beat (${prevT}) — events out of order`,
|
|
107
|
+
"events follow the recording; reordering needs a re-capture, not a swap here",
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
prevT = e.tMs;
|
|
111
|
+
|
|
112
|
+
if (e.tMs < 0) err(`${p}.tMs`, `negative tMs ${e.tMs}`);
|
|
113
|
+
if (e.tMs > comp.durationMs)
|
|
114
|
+
err(
|
|
115
|
+
`${p}.tMs`,
|
|
116
|
+
`tMs ${e.tMs} exceeds composition durationMs ${comp.durationMs}`,
|
|
117
|
+
"raise durationMs or remove the beat",
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
// kind-specific required fields (editability invariants)
|
|
121
|
+
const need = reqField[e.kind];
|
|
122
|
+
if (need && (e as Record<string, unknown>)[need] == null)
|
|
123
|
+
err(
|
|
124
|
+
`${p}.${need}`,
|
|
125
|
+
`${e.kind} beat is missing "${need}"`,
|
|
126
|
+
`restore ${need} from the capture`,
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
// --- zoom decision ---
|
|
130
|
+
const z = e.zoom;
|
|
131
|
+
if (!z) {
|
|
132
|
+
err(
|
|
133
|
+
`${p}.zoom`,
|
|
134
|
+
"missing zoom decision",
|
|
135
|
+
"every beat needs a zoom { enabled, scale, center, inAtMs }",
|
|
136
|
+
);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
// inAtMs must precede the action (the zoom-in ramps in before the beat lands)
|
|
140
|
+
if (z.inAtMs < 0) err(`${p}.zoom.inAtMs`, `negative inAtMs ${z.inAtMs}`, "clamp to 0");
|
|
141
|
+
if (z.inAtMs > e.tMs)
|
|
142
|
+
err(
|
|
143
|
+
`${p}.zoom.inAtMs`,
|
|
144
|
+
`inAtMs ${z.inAtMs} is AFTER the action at tMs ${e.tMs} — the zoom would arrive late`,
|
|
145
|
+
`set inAtMs = tMs − cursor.zoomInMs (= ${Math.max(0, e.tMs - comp.cursor.zoomInMs)})`,
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
if (z.enabled) {
|
|
149
|
+
// scale must zoom IN (≥ rest). Below rest shows MORE than the framed
|
|
150
|
+
// video → backdrop dead-space; that's never what an enabled zoom wants.
|
|
151
|
+
if (z.scale < rest - 1e-6)
|
|
152
|
+
err(
|
|
153
|
+
`${p}.zoom.scale`,
|
|
154
|
+
`scale ${z.scale.toFixed(3)} is below rest ${rest.toFixed(3)} — that zooms OUT past the frame (dead space)`,
|
|
155
|
+
`raise to ≥ ${rest.toFixed(2)}, or set zoom.enabled=false for a full-view beat`,
|
|
156
|
+
);
|
|
157
|
+
else if (z.scale <= rest + 1e-6)
|
|
158
|
+
warn(
|
|
159
|
+
`${p}.zoom.scale`,
|
|
160
|
+
`scale ${z.scale.toFixed(3)} ≈ rest — zoom is enabled but does nothing visible`,
|
|
161
|
+
"raise scale to actually zoom, or set enabled=false",
|
|
162
|
+
);
|
|
163
|
+
if (z.scale > maxScale)
|
|
164
|
+
warn(
|
|
165
|
+
`${p}.zoom.scale`,
|
|
166
|
+
`scale ${z.scale.toFixed(2)} exceeds the soft cap ${maxScale} — pixels will soften`,
|
|
167
|
+
`clamp toward ${maxScale.toFixed(1)} unless the detail truly needs it`,
|
|
168
|
+
);
|
|
169
|
+
// center inside the video (clampCenter will pull it in, but an off-video
|
|
170
|
+
// center usually means a stale/hand-miscomputed value)
|
|
171
|
+
if (z.center.x < 0 || z.center.x > vW || z.center.y < 0 || z.center.y > vH)
|
|
172
|
+
warn(
|
|
173
|
+
`${p}.zoom.center`,
|
|
174
|
+
`center (${z.center.x.toFixed(0)},${z.center.y.toFixed(0)}) is outside the video ${vW}x${vH}`,
|
|
175
|
+
e.bbox
|
|
176
|
+
? "use the bbox center: { x: bbox.x+bbox.w/2, y: bbox.y+bbox.h/2 }"
|
|
177
|
+
: "point at the on-screen target (video-px)",
|
|
178
|
+
);
|
|
179
|
+
// enabling zoom on a no-bbox beat means an invented center/scale — fine,
|
|
180
|
+
// but flag it so the refine loop knows it's not bbox-derived.
|
|
181
|
+
if (!e.bbox)
|
|
182
|
+
warn(
|
|
183
|
+
`${p}.zoom`,
|
|
184
|
+
"zoom enabled on a beat with no bbox — center/scale are hand-set, not bbox-fit",
|
|
185
|
+
"double-check the center frames the intended region",
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// tail: the composition must outlast the last action (+ a little settle)
|
|
191
|
+
if (comp.durationMs < lastEnd)
|
|
192
|
+
err(
|
|
193
|
+
"durationMs",
|
|
194
|
+
`durationMs ${comp.durationMs} ends before the last action (${lastEnd})`,
|
|
195
|
+
`raise to ≥ ${Math.round(lastEnd + comp.cursor.holdMs + comp.cursor.zoomOutMs)}`,
|
|
196
|
+
);
|
|
197
|
+
else if (comp.durationMs < lastEnd + comp.cursor.zoomOutMs)
|
|
198
|
+
warn(
|
|
199
|
+
"durationMs",
|
|
200
|
+
`only ${comp.durationMs - lastEnd}ms after the last action — the final zoom-out may be cut`,
|
|
201
|
+
`allow ≥ ${comp.cursor.zoomOutMs}ms (cursor.zoomOutMs) of tail`,
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
// --- CAPTURE-LOCKED: action tMs must match the recording ---
|
|
205
|
+
// The video is temporal: a beat's tMs is WHEN it is visible in the capture.
|
|
206
|
+
// A render-only edit cannot move that, so a drifted tMs would fire the
|
|
207
|
+
// overlay off the on-screen action. (Cinematic timing — inAtMs, holdMs,
|
|
208
|
+
// zoom ramps, the intro travel and the tail — IS freely editable; only the
|
|
209
|
+
// action anchor tMs is locked.)
|
|
210
|
+
if (opts.captureLog) {
|
|
211
|
+
const log = opts.captureLog;
|
|
212
|
+
if (log.events.length !== events.length)
|
|
213
|
+
warn(
|
|
214
|
+
"events",
|
|
215
|
+
`composition has ${events.length} beats but the capture log has ${log.events.length} — added/removed actions need a re-capture`,
|
|
216
|
+
"re-capture to change the choreography; edit only renders the existing recording",
|
|
217
|
+
);
|
|
218
|
+
const n = Math.min(log.events.length, events.length);
|
|
219
|
+
for (let i = 0; i < n; i++) {
|
|
220
|
+
const lt = log.events[i]!.tMs;
|
|
221
|
+
const ct = events[i]!.tMs;
|
|
222
|
+
if (Math.abs(lt - ct) > 1)
|
|
223
|
+
err(
|
|
224
|
+
`events[${i}].tMs`,
|
|
225
|
+
`tMs ${ct} drifted from the captured ${lt} — an action's tMs is capture-locked (the video shows it at ${lt}ms)`,
|
|
226
|
+
`restore tMs to ${lt}; to retime the action itself, re-capture`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return issues;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Format issues for a human/agent log. Errors first. */
|
|
235
|
+
export function formatIssues(issues: CompositionIssue[]): string {
|
|
236
|
+
if (!issues.length) return "composition OK (0 issues)";
|
|
237
|
+
const order = (s: string) => (s === "error" ? 0 : 1);
|
|
238
|
+
return issues
|
|
239
|
+
.slice()
|
|
240
|
+
.sort((a, b) => order(a.severity) - order(b.severity))
|
|
241
|
+
.map(
|
|
242
|
+
(i) => ` [${i.severity}] ${i.path}: ${i.message}${i.fix ? `\n fix: ${i.fix}` : ""}`,
|
|
243
|
+
)
|
|
244
|
+
.join("\n");
|
|
245
|
+
}
|