@tandryukha/aidemo 0.8.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +734 -48
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +244 -36
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +711 -26
- package/src/compose.ts +1318 -101
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +19 -4
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +224 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +208 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +640 -0
- package/src/inspect.ts +316 -0
- package/src/lint.ts +673 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +5 -1
- package/src/mcp/server.ts +468 -27
- package/src/player.ts +1286 -41
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/qa.ts +295 -0
- package/src/recorder.ts +316 -21
- package/src/safe-regex.ts +36 -0
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +796 -22
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/cursor-overlay.ts
CHANGED
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
* like the baked one.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
import { type Key, num, piecewiseExpr, simplifyKeys } from "./expr.js";
|
|
18
|
+
|
|
17
19
|
/** A cursor sample already mapped into final content time + output pixels. */
|
|
18
20
|
export interface CursorPoint {
|
|
19
21
|
/** Content-time seconds. */
|
|
@@ -34,121 +36,11 @@ export interface HideWindow {
|
|
|
34
36
|
* up (ENOMEM) somewhere past ~100 flat terms — measured ~80 OK / 120 fail on a
|
|
35
37
|
* stock build. X(t) and Y(t) are INDEPENDENT overlay expressions, so this budget
|
|
36
38
|
* is per axis; keeping it well under the cliff (and small) also protects weaker
|
|
37
|
-
* ffmpeg builds. The path is simplified to fit (
|
|
39
|
+
* ffmpeg builds. The path is simplified to fit (expr.ts simplifyKeys) — cheap, because
|
|
38
40
|
* the player's glides are eased S-curves that RDP collapses to a few keys each.
|
|
39
41
|
*/
|
|
40
42
|
const KEY_BUDGET = 48;
|
|
41
43
|
|
|
42
|
-
interface Key {
|
|
43
|
-
t: number;
|
|
44
|
-
v: number;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function num(n: number): string {
|
|
48
|
-
const s = n.toFixed(3).replace(/\.?0+$/, "");
|
|
49
|
-
return s === "" || s === "-" ? "0" : s;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
/** Linear-interpolation (vertical) error at `p` against the a→b chord. */
|
|
53
|
-
function chordError(a: Key, b: Key, p: Key): number {
|
|
54
|
-
const dt = b.t - a.t;
|
|
55
|
-
const vAt = dt < 1e-9 ? a.v : a.v + ((b.v - a.v) * (p.t - a.t)) / dt;
|
|
56
|
-
return Math.abs(p.v - vAt);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Ramer–Douglas–Peucker on a value-over-time polyline, using vertical (linear-
|
|
61
|
-
* interpolation) error in the value's own units (pixels) — so `eps` is exactly
|
|
62
|
-
* "max pixels the drawn cursor may drift from the recorded path". Keeps the
|
|
63
|
-
* points where the curve actually bends; a straight or held stretch collapses to
|
|
64
|
-
* its endpoints. Iterative to avoid deep recursion on long inputs.
|
|
65
|
-
*/
|
|
66
|
-
function rdp(keys: Key[], eps: number): Key[] {
|
|
67
|
-
const n = keys.length;
|
|
68
|
-
if (n <= 2) return keys.slice();
|
|
69
|
-
const keep = new Uint8Array(n);
|
|
70
|
-
keep[0] = 1;
|
|
71
|
-
keep[n - 1] = 1;
|
|
72
|
-
const stack: Array<[number, number]> = [[0, n - 1]];
|
|
73
|
-
while (stack.length) {
|
|
74
|
-
const [lo, hi] = stack.pop()!;
|
|
75
|
-
if (hi - lo < 2) continue;
|
|
76
|
-
let maxErr = -1;
|
|
77
|
-
let idx = -1;
|
|
78
|
-
for (let i = lo + 1; i < hi; i++) {
|
|
79
|
-
const e = chordError(keys[lo], keys[hi], keys[i]);
|
|
80
|
-
if (e > maxErr) {
|
|
81
|
-
maxErr = e;
|
|
82
|
-
idx = i;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
if (maxErr > eps && idx >= 0) {
|
|
86
|
-
keep[idx] = 1;
|
|
87
|
-
stack.push([lo, idx], [idx, hi]);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
const out: Key[] = [];
|
|
91
|
-
for (let i = 0; i < n; i++) if (keep[i]) out.push(keys[i]);
|
|
92
|
-
return out;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Simplify one axis to at most KEY_BUDGET keys: start at a sub-pixel epsilon and
|
|
97
|
-
* grow it until the point count fits, so a busy path just drifts a pixel or two
|
|
98
|
-
* more rather than blowing the expression budget.
|
|
99
|
-
*/
|
|
100
|
-
function simplifyAxis(keys: Key[]): Key[] {
|
|
101
|
-
if (keys.length <= KEY_BUDGET) {
|
|
102
|
-
const r = rdp(keys, 1.2);
|
|
103
|
-
if (r.length <= KEY_BUDGET) return r;
|
|
104
|
-
}
|
|
105
|
-
let eps = 1.2;
|
|
106
|
-
for (let i = 0; i < 24; i++) {
|
|
107
|
-
const r = rdp(keys, eps);
|
|
108
|
-
if (r.length <= KEY_BUDGET) return r;
|
|
109
|
-
eps *= 1.6;
|
|
110
|
-
}
|
|
111
|
-
// Backstop: uniform thin (should never be reached for a real cursor path).
|
|
112
|
-
const step = Math.ceil(keys.length / KEY_BUDGET);
|
|
113
|
-
const out = keys.filter((_, i) => i % step === 0);
|
|
114
|
-
if (out[out.length - 1] !== keys[keys.length - 1]) out.push(keys[keys.length - 1]);
|
|
115
|
-
return out;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Piecewise-linear expression over overlay's `t` (seconds): holds the first
|
|
120
|
-
* value before the first key, linearly interpolates between keys, holds the last
|
|
121
|
-
* value after the last.
|
|
122
|
-
*
|
|
123
|
-
* Emitted as a FLAT sum of half-open gated segments — NOT nested `if`s. A path
|
|
124
|
-
* has dozens–hundreds of keys, and ffmpeg's expression parser blows its nesting
|
|
125
|
-
* limit ("Missing ')' or too many args") a hundred `if`s deep (zoom.ts gets away
|
|
126
|
-
* with nested `if` only because it has a handful of keys). Each segment
|
|
127
|
-
* [t_i, t_{i+1}) contributes `gte(t,t_i)*lt(t,t_{i+1})*(v_i + slope*(t-t_i))`;
|
|
128
|
-
* the gates are disjoint half-open intervals so exactly one term is live at a
|
|
129
|
-
* time. Depth stays ~3 regardless of key count.
|
|
130
|
-
*/
|
|
131
|
-
function piecewiseLinear(keys: Key[]): string {
|
|
132
|
-
if (keys.length === 1) return num(keys[0].v);
|
|
133
|
-
const first = keys[0];
|
|
134
|
-
const last = keys[keys.length - 1];
|
|
135
|
-
const terms: string[] = [`lt(t,${num(first.t)})*${num(first.v)}`];
|
|
136
|
-
for (let i = 0; i < keys.length - 1; i++) {
|
|
137
|
-
const a = keys[i];
|
|
138
|
-
const b = keys[i + 1];
|
|
139
|
-
const dt = b.t - a.t;
|
|
140
|
-
const gate = `gte(t,${num(a.t)})*lt(t,${num(b.t)})`;
|
|
141
|
-
if (a.v === b.v || dt < 1e-6) {
|
|
142
|
-
terms.push(`${gate}*${num(a.v)}`);
|
|
143
|
-
} else {
|
|
144
|
-
const slope = (b.v - a.v) / dt;
|
|
145
|
-
terms.push(`${gate}*(${num(a.v)}+(${num(slope)})*(t-${num(a.t)}))`);
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
terms.push(`gte(t,${num(last.t)})*${num(last.v)}`);
|
|
149
|
-
return terms.join("+");
|
|
150
|
-
}
|
|
151
|
-
|
|
152
44
|
/**
|
|
153
45
|
* Build the `-filter_complex` string overlaying the cursor PNG (input 1) onto
|
|
154
46
|
* the content (input 0) along `rawPts`, hidden during `hideWindows`. Returns
|
|
@@ -170,8 +62,10 @@ export function buildCursorFilter(
|
|
|
170
62
|
}
|
|
171
63
|
// Simplify each axis independently (X and Y are separate expressions) so each
|
|
172
64
|
// stays under the ffmpeg expression budget.
|
|
173
|
-
const
|
|
174
|
-
|
|
65
|
+
const axis = (vals: Key[]): string =>
|
|
66
|
+
piecewiseExpr(simplifyKeys(vals, KEY_BUDGET, 1.2), "t", "linear");
|
|
67
|
+
const X = axis(mono.map((p) => ({ t: p.t, v: p.x })));
|
|
68
|
+
const Y = axis(mono.map((p) => ({ t: p.t, v: p.y })));
|
|
175
69
|
|
|
176
70
|
// enable is single-quoted so the commas inside between()/not() are literal,
|
|
177
71
|
// not filtergraph separators (same reason zoom single-quotes its expressions).
|
package/src/cursor.ts
CHANGED
|
@@ -22,7 +22,10 @@ const CURSOR_STROKE = "#111111";
|
|
|
22
22
|
*/
|
|
23
23
|
export async function renderCursorPng(
|
|
24
24
|
outPath: string,
|
|
25
|
-
sizePx: number
|
|
25
|
+
sizePx: number,
|
|
26
|
+
style: "arrow" | "dot" = "arrow",
|
|
27
|
+
color = "rgba(255,90,95,.85)",
|
|
28
|
+
look: { outline?: string; outlineWidth?: number; halo?: boolean } = {}
|
|
26
29
|
): Promise<void> {
|
|
27
30
|
// Logical canvas is 32 units (24 arrow + shadow headroom); scale to sizePx.
|
|
28
31
|
const LOGICAL = 32;
|
|
@@ -38,12 +41,24 @@ export async function renderCursorPng(
|
|
|
38
41
|
`<!doctype html><html><head><meta charset="utf-8"><style>
|
|
39
42
|
html,body{margin:0;padding:0;background:transparent;}
|
|
40
43
|
svg{position:absolute;left:0;top:0;
|
|
41
|
-
filter
|
|
44
|
+
filter:${
|
|
45
|
+
look.halo
|
|
46
|
+
? "drop-shadow(0 0 3px rgba(0,0,0,.65)) drop-shadow(0 0 7px rgba(0,0,0,.45))"
|
|
47
|
+
: "drop-shadow(0 1px 2px rgba(0,0,0,.45))"
|
|
48
|
+
};}
|
|
42
49
|
</style></head><body>
|
|
43
50
|
<svg width="${LOGICAL}" height="${LOGICAL}" viewBox="0 0 ${LOGICAL} ${LOGICAL}"
|
|
44
51
|
xmlns="http://www.w3.org/2000/svg">
|
|
45
|
-
|
|
46
|
-
|
|
52
|
+
${
|
|
53
|
+
style === "dot"
|
|
54
|
+
? `<circle cx="16" cy="16" r="9" fill="${color}" stroke="${
|
|
55
|
+
look.outline ?? "rgba(255,255,255,.9)"
|
|
56
|
+
}" stroke-width="${look.outlineWidth ?? 2}"/>`
|
|
57
|
+
: `<path d="${CURSOR_ARROW_PATH}" fill="${CURSOR_FILL}"
|
|
58
|
+
stroke="${look.outline ?? CURSOR_STROKE}" stroke-width="${
|
|
59
|
+
look.outlineWidth ?? 1.3
|
|
60
|
+
}" stroke-linejoin="round"/>`
|
|
61
|
+
}
|
|
47
62
|
</svg>
|
|
48
63
|
</body></html>`,
|
|
49
64
|
{ waitUntil: "load" }
|
package/src/distribute.ts
CHANGED
|
@@ -20,7 +20,8 @@ import { platform, release } from "node:os";
|
|
|
20
20
|
import { ENGINE_ROOT, engineVersion, openAiBaseUrl, ttsProvider } from "./config.js";
|
|
21
21
|
import { localTtsStatus } from "./voice-local.js";
|
|
22
22
|
import { Project } from "./project.js";
|
|
23
|
-
import { STARTER_BRIEF, STARTER_STORYBOARD } from "./starter.js";
|
|
23
|
+
import { STARTER_BRIEF, STARTER_STORYBOARD, briefFromInspect, storyboardFromInspect } from "./starter.js";
|
|
24
|
+
import { inspectPage } from "./inspect.js";
|
|
24
25
|
import { ensureDir, exists, ok, step, fail, log } from "./util.js";
|
|
25
26
|
|
|
26
27
|
const execFileAsync = promisify(execFile);
|
|
@@ -239,7 +240,14 @@ async function registerMcpServer(
|
|
|
239
240
|
export async function scaffoldDemo(
|
|
240
241
|
baseDir: string,
|
|
241
242
|
name: string,
|
|
242
|
-
opts: {
|
|
243
|
+
opts: {
|
|
244
|
+
force?: boolean;
|
|
245
|
+
/** Draft from a live page: inspect it and turn headings/selectors into scenes. */
|
|
246
|
+
fromUrl?: string;
|
|
247
|
+
headed?: boolean;
|
|
248
|
+
profileDir?: string;
|
|
249
|
+
viewport?: { width: number; height: number };
|
|
250
|
+
} = {}
|
|
243
251
|
): Promise<string> {
|
|
244
252
|
const dir = resolve(baseDir, "demos", name);
|
|
245
253
|
if ((await exists(dir)) && !opts.force) {
|
|
@@ -248,6 +256,23 @@ export async function scaffoldDemo(
|
|
|
248
256
|
}
|
|
249
257
|
const project = new Project(dir);
|
|
250
258
|
await project.ensureDirs();
|
|
259
|
+
if (opts.fromUrl) {
|
|
260
|
+
step(`Inspecting ${opts.fromUrl} for a first draft`);
|
|
261
|
+
const res = await inspectPage({
|
|
262
|
+
url: opts.fromUrl,
|
|
263
|
+
headless: !opts.headed,
|
|
264
|
+
profileDir: opts.profileDir,
|
|
265
|
+
viewport: opts.viewport,
|
|
266
|
+
screenshotPath: project.p("logs", "inspect-init.png"),
|
|
267
|
+
});
|
|
268
|
+
await fs.writeFile(project.p("logs", "inspect-init.json"), JSON.stringify(res, null, 2));
|
|
269
|
+
await fs.writeFile(project.p("input", "brief.md"), briefFromInspect(name, res));
|
|
270
|
+
await fs.writeFile(project.storyboardPath, storyboardFromInspect(name, res));
|
|
271
|
+
ok(
|
|
272
|
+
`scaffolded demos/${name}/ from ${res.finalUrl || res.url} — ${res.headings.length} heading(s), ${res.elements.length} element(s) seen`
|
|
273
|
+
);
|
|
274
|
+
return dir;
|
|
275
|
+
}
|
|
251
276
|
await fs.writeFile(project.p("input", "brief.md"), STARTER_BRIEF(name));
|
|
252
277
|
await fs.writeFile(project.storyboardPath, STARTER_STORYBOARD(name));
|
|
253
278
|
ok(`scaffolded demos/${name}/`);
|
|
@@ -493,10 +518,18 @@ export async function fileFeedback(
|
|
|
493
518
|
*/
|
|
494
519
|
export async function feedback(
|
|
495
520
|
dir: string | undefined,
|
|
496
|
-
opts: {
|
|
521
|
+
opts: {
|
|
522
|
+
web?: boolean;
|
|
523
|
+
dryRun?: boolean;
|
|
524
|
+
title?: string;
|
|
525
|
+
description?: string;
|
|
526
|
+
} = {}
|
|
497
527
|
): Promise<void> {
|
|
498
528
|
const demoDir = dir ? targetOf(dir) : null;
|
|
499
|
-
const ctx = await buildFeedback(demoDir
|
|
529
|
+
const ctx = await buildFeedback(demoDir, {
|
|
530
|
+
title: opts.title,
|
|
531
|
+
description: opts.description,
|
|
532
|
+
});
|
|
500
533
|
|
|
501
534
|
if (opts.dryRun) {
|
|
502
535
|
step(`feedback (dry run) → ${UPSTREAM_REPO}`);
|
package/src/expr.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for building ffmpeg filter *expressions* from keyframes.
|
|
3
|
+
*
|
|
4
|
+
* ffmpeg's expression evaluator has two independent cliffs, and both bite once
|
|
5
|
+
* a demo gets long enough:
|
|
6
|
+
*
|
|
7
|
+
* 1. **Parser nesting.** A chain of nested `if(...)`s dies with
|
|
8
|
+
* `Missing ')' or too many args` about a hundred levels deep. A 13-scene
|
|
9
|
+
* take with ~40 focus points hit this in the zoom pass (issue #36).
|
|
10
|
+
* 2. **AST size.** Past roughly a hundred flat terms the evaluator allocates
|
|
11
|
+
* until it fails (ENOMEM) — measured ~80 OK / 120 fail on a stock build.
|
|
12
|
+
*
|
|
13
|
+
* So every expression here is emitted as a FLAT sum of disjoint half-open
|
|
14
|
+
* gated segments (depth ~3 regardless of key count), and every channel is
|
|
15
|
+
* first simplified (Ramer–Douglas–Peucker) to a key budget that stays well
|
|
16
|
+
* under cliff 2.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export interface Key {
|
|
20
|
+
/** Seconds. */
|
|
21
|
+
t: number;
|
|
22
|
+
v: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Format a number for an ffmpeg expression (no trailing zeros, no `NaN`). */
|
|
26
|
+
export function num(n: number): string {
|
|
27
|
+
const s = n.toFixed(4).replace(/\.?0+$/, "");
|
|
28
|
+
return s === "" || s === "-" ? "0" : s;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Linear-interpolation (vertical) error at `p` against the a→b chord. */
|
|
32
|
+
function chordError(a: Key, b: Key, p: Key): number {
|
|
33
|
+
const dt = b.t - a.t;
|
|
34
|
+
const vAt = dt < 1e-9 ? a.v : a.v + ((b.v - a.v) * (p.t - a.t)) / dt;
|
|
35
|
+
return Math.abs(p.v - vAt);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Ramer–Douglas–Peucker on a value-over-time polyline, using vertical error in
|
|
40
|
+
* the value's own units — so `eps` is exactly "how far the drawn channel may
|
|
41
|
+
* drift from the planned one". Keeps the points where the curve actually
|
|
42
|
+
* bends; a straight or held stretch collapses to its endpoints. Iterative to
|
|
43
|
+
* avoid deep recursion on long inputs.
|
|
44
|
+
*/
|
|
45
|
+
export function rdp(keys: Key[], eps: number): Key[] {
|
|
46
|
+
const n = keys.length;
|
|
47
|
+
if (n <= 2) return keys.slice();
|
|
48
|
+
const keep = new Uint8Array(n);
|
|
49
|
+
keep[0] = 1;
|
|
50
|
+
keep[n - 1] = 1;
|
|
51
|
+
const stack: Array<[number, number]> = [[0, n - 1]];
|
|
52
|
+
while (stack.length) {
|
|
53
|
+
const [lo, hi] = stack.pop()!;
|
|
54
|
+
if (hi - lo < 2) continue;
|
|
55
|
+
let maxErr = -1;
|
|
56
|
+
let idx = -1;
|
|
57
|
+
for (let i = lo + 1; i < hi; i++) {
|
|
58
|
+
const e = chordError(keys[lo], keys[hi], keys[i]);
|
|
59
|
+
if (e > maxErr) {
|
|
60
|
+
maxErr = e;
|
|
61
|
+
idx = i;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (maxErr > eps && idx >= 0) {
|
|
65
|
+
keep[idx] = 1;
|
|
66
|
+
stack.push([lo, idx], [idx, hi]);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const out: Key[] = [];
|
|
70
|
+
for (let i = 0; i < n; i++) if (keep[i]) out.push(keys[i]);
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Simplify one channel to at most `budget` keys: start at `eps` and grow it
|
|
76
|
+
* until the point count fits, so a busy channel just drifts slightly more
|
|
77
|
+
* rather than blowing the expression budget.
|
|
78
|
+
*/
|
|
79
|
+
export function simplifyKeys(keys: Key[], budget: number, eps: number): Key[] {
|
|
80
|
+
if (keys.length <= budget) {
|
|
81
|
+
const r = rdp(keys, eps);
|
|
82
|
+
if (r.length <= budget) return r;
|
|
83
|
+
}
|
|
84
|
+
let e = eps;
|
|
85
|
+
for (let i = 0; i < 24; i++) {
|
|
86
|
+
const r = rdp(keys, e);
|
|
87
|
+
if (r.length <= budget) return r;
|
|
88
|
+
e *= 1.6;
|
|
89
|
+
}
|
|
90
|
+
// Backstop: uniform thin (should never be reached for a real channel).
|
|
91
|
+
const step = Math.ceil(keys.length / budget);
|
|
92
|
+
const out = keys.filter((_, i) => i % step === 0);
|
|
93
|
+
if (out[out.length - 1] !== keys[keys.length - 1]) out.push(keys[keys.length - 1]);
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
type Interp = "linear" | "smooth";
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Piecewise expression over the time expression `T` (seconds): holds the first
|
|
101
|
+
* value before the first key, interpolates between keys (linearly, or with a
|
|
102
|
+
* smoothstep ease), holds the last value after the last key.
|
|
103
|
+
*
|
|
104
|
+
* Emitted as a flat sum of half-open gated terms — see the module header for
|
|
105
|
+
* why nesting `if`s is not an option.
|
|
106
|
+
*/
|
|
107
|
+
export function piecewiseExpr(keys: Key[], T: string, interp: Interp): string {
|
|
108
|
+
if (keys.length === 0) return "0";
|
|
109
|
+
if (keys.length === 1) return num(keys[0].v);
|
|
110
|
+
const first = keys[0];
|
|
111
|
+
const last = keys[keys.length - 1];
|
|
112
|
+
const terms: string[] = [`lt(${T},${num(first.t)})*${num(first.v)}`];
|
|
113
|
+
for (let i = 0; i < keys.length - 1; i++) {
|
|
114
|
+
const a = keys[i];
|
|
115
|
+
const b = keys[i + 1];
|
|
116
|
+
const dt = b.t - a.t;
|
|
117
|
+
const gate = `gte(${T},${num(a.t)})*lt(${T},${num(b.t)})`;
|
|
118
|
+
if (a.v === b.v || dt < 1e-6) {
|
|
119
|
+
terms.push(`${gate}*${num(a.v)}`);
|
|
120
|
+
} else if (interp === "linear") {
|
|
121
|
+
const slope = (b.v - a.v) / dt;
|
|
122
|
+
terms.push(`${gate}*(${num(a.v)}+(${num(slope)})*(${T}-${num(a.t)}))`);
|
|
123
|
+
} else {
|
|
124
|
+
const p = `((${T}-${num(a.t)})/${num(dt)})`;
|
|
125
|
+
terms.push(`${gate}*(${num(a.v)}+(${num(b.v - a.v)})*${p}*${p}*(3-2*${p}))`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
terms.push(`gte(${T},${num(last.t)})*${num(last.v)}`);
|
|
129
|
+
return terms.join("+");
|
|
130
|
+
}
|
package/src/ffmpeg.ts
CHANGED
|
@@ -81,3 +81,227 @@ export function probeDurationMs(file: string): Promise<number> {
|
|
|
81
81
|
});
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Length (ms) of the *flat* tail of a video — the trailing run of frames that
|
|
87
|
+
* are a single solid color.
|
|
88
|
+
*
|
|
89
|
+
* Compose freeze-holds a scene's last frame when the narration outruns the
|
|
90
|
+
* recorded action. When the scene ends right before a `goto`, that last frame
|
|
91
|
+
* is the browser's white pre-paint, and the hold clones *white* for as long as
|
|
92
|
+
* the narration runs (issue #36: 37 near-blank seconds out of 209). Trimming
|
|
93
|
+
* the flat tail first makes the hold clone the last frame that actually shows
|
|
94
|
+
* the app.
|
|
95
|
+
*
|
|
96
|
+
* Flatness is `YHIGH - YLOW` from `signalstats` — the spread between the luma
|
|
97
|
+
* 90th and 10th percentiles. Percentiles rather than min/max so a cursor arrow
|
|
98
|
+
* or a stray antialiased pixel doesn't make a blank page look like content;
|
|
99
|
+
* `YDEV` would be the natural measure but many ffmpeg builds don't emit it.
|
|
100
|
+
*
|
|
101
|
+
* Scans at most `windowMs` from the end; returns 0 when the tail has real
|
|
102
|
+
* content, or when the probe fails for any reason (best-effort polish, never a
|
|
103
|
+
* reason to fail a compose).
|
|
104
|
+
*/
|
|
105
|
+
export function probeFlatTailMs(
|
|
106
|
+
file: string,
|
|
107
|
+
durMs: number,
|
|
108
|
+
windowMs = 4000
|
|
109
|
+
): Promise<number> {
|
|
110
|
+
const start = Math.max(0, durMs - windowMs) / 1000;
|
|
111
|
+
return new Promise((resolvePromise) => {
|
|
112
|
+
const proc = spawn(
|
|
113
|
+
"ffmpeg",
|
|
114
|
+
[
|
|
115
|
+
"-hide_banner",
|
|
116
|
+
"-nostats",
|
|
117
|
+
"-ss",
|
|
118
|
+
start.toFixed(3),
|
|
119
|
+
"-i",
|
|
120
|
+
file,
|
|
121
|
+
"-vf",
|
|
122
|
+
"signalstats,metadata=print:file=-",
|
|
123
|
+
"-f",
|
|
124
|
+
"null",
|
|
125
|
+
"-",
|
|
126
|
+
],
|
|
127
|
+
{ stdio: ["ignore", "pipe", "ignore"] }
|
|
128
|
+
);
|
|
129
|
+
let out = "";
|
|
130
|
+
proc.stdout.on("data", (d) => (out += d.toString()));
|
|
131
|
+
proc.on("error", () => resolvePromise(0));
|
|
132
|
+
proc.on("close", () => {
|
|
133
|
+
// The metadata muxer emits a header line per frame followed by one line
|
|
134
|
+
// per key:
|
|
135
|
+
// frame:12 pts:... pts_time:0.4
|
|
136
|
+
// lavfi.signalstats.YLOW=16
|
|
137
|
+
// ...
|
|
138
|
+
const frames: Array<{ t: number; spread: number }> = [];
|
|
139
|
+
let t = NaN;
|
|
140
|
+
let low = NaN;
|
|
141
|
+
let high = NaN;
|
|
142
|
+
const flush = (): void => {
|
|
143
|
+
if (!Number.isNaN(t) && !Number.isNaN(low) && !Number.isNaN(high)) {
|
|
144
|
+
frames.push({ t, spread: high - low });
|
|
145
|
+
}
|
|
146
|
+
low = NaN;
|
|
147
|
+
high = NaN;
|
|
148
|
+
};
|
|
149
|
+
for (const line of out.split("\n")) {
|
|
150
|
+
const ts = /pts_time:\s*([\d.]+)/.exec(line);
|
|
151
|
+
if (ts) {
|
|
152
|
+
flush();
|
|
153
|
+
t = parseFloat(ts[1]);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
const lo = /YLOW=\s*([\d.]+)/.exec(line);
|
|
157
|
+
if (lo) low = parseFloat(lo[1]);
|
|
158
|
+
const hi = /YHIGH=\s*([\d.]+)/.exec(line);
|
|
159
|
+
if (hi) high = parseFloat(hi[1]);
|
|
160
|
+
}
|
|
161
|
+
flush();
|
|
162
|
+
if (frames.length === 0) return resolvePromise(0);
|
|
163
|
+
// Walk back while frames stay flat. A real page — even a mostly-empty
|
|
164
|
+
// one — has chrome, text and edges, so its spread is far above this.
|
|
165
|
+
const FLAT_SPREAD = 4;
|
|
166
|
+
let i = frames.length - 1;
|
|
167
|
+
if (frames[i].spread > FLAT_SPREAD) return resolvePromise(0);
|
|
168
|
+
while (i > 0 && frames[i - 1].spread <= FLAT_SPREAD) i--;
|
|
169
|
+
resolvePromise(Math.max(0, Math.round(durMs - (start + frames[i].t) * 1000)));
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Mirror of `probeFlatTailMs` for the HEAD of a file: how many ms at the start
|
|
176
|
+
* are a flat, contentless frame (the browser's white pre-paint after a `goto`).
|
|
177
|
+
* Used to open the video on the product instead of on a blank page, and to pick
|
|
178
|
+
* a poster frame that isn't a white rectangle (issue #47). Best-effort: any
|
|
179
|
+
* probe failure resolves to 0.
|
|
180
|
+
*/
|
|
181
|
+
export function probeFlatLeadMs(
|
|
182
|
+
file: string,
|
|
183
|
+
windowMs = 4000,
|
|
184
|
+
startMs = 0
|
|
185
|
+
): Promise<number> {
|
|
186
|
+
return new Promise((resolvePromise) => {
|
|
187
|
+
const proc = spawn(
|
|
188
|
+
"ffmpeg",
|
|
189
|
+
[
|
|
190
|
+
"-hide_banner",
|
|
191
|
+
"-nostats",
|
|
192
|
+
...(startMs > 0 ? ["-ss", (startMs / 1000).toFixed(3)] : []),
|
|
193
|
+
"-i",
|
|
194
|
+
file,
|
|
195
|
+
"-t",
|
|
196
|
+
(windowMs / 1000).toFixed(3),
|
|
197
|
+
"-vf",
|
|
198
|
+
"signalstats,metadata=print:file=-",
|
|
199
|
+
"-f",
|
|
200
|
+
"null",
|
|
201
|
+
"-",
|
|
202
|
+
],
|
|
203
|
+
{ stdio: ["ignore", "pipe", "ignore"] }
|
|
204
|
+
);
|
|
205
|
+
let out = "";
|
|
206
|
+
proc.stdout.on("data", (d) => (out += d.toString()));
|
|
207
|
+
proc.on("error", () => resolvePromise(0));
|
|
208
|
+
proc.on("close", () => {
|
|
209
|
+
const frames: Array<{ t: number; spread: number }> = [];
|
|
210
|
+
let t = NaN;
|
|
211
|
+
let low = NaN;
|
|
212
|
+
let high = NaN;
|
|
213
|
+
const flush = (): void => {
|
|
214
|
+
if (!Number.isNaN(t) && !Number.isNaN(low) && !Number.isNaN(high)) {
|
|
215
|
+
frames.push({ t, spread: high - low });
|
|
216
|
+
}
|
|
217
|
+
low = NaN;
|
|
218
|
+
high = NaN;
|
|
219
|
+
};
|
|
220
|
+
for (const line of out.split("\n")) {
|
|
221
|
+
const ts = /pts_time:\s*([\d.]+)/.exec(line);
|
|
222
|
+
if (ts) {
|
|
223
|
+
flush();
|
|
224
|
+
t = parseFloat(ts[1]);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
const lo = /YLOW=\s*([\d.]+)/.exec(line);
|
|
228
|
+
if (lo) low = parseFloat(lo[1]);
|
|
229
|
+
const hi = /YHIGH=\s*([\d.]+)/.exec(line);
|
|
230
|
+
if (hi) high = parseFloat(hi[1]);
|
|
231
|
+
}
|
|
232
|
+
flush();
|
|
233
|
+
const FLAT_SPREAD = 4;
|
|
234
|
+
if (!frames.length || frames[0].spread > FLAT_SPREAD) return resolvePromise(0);
|
|
235
|
+
let i = 0;
|
|
236
|
+
while (i + 1 < frames.length && frames[i + 1].spread <= FLAT_SPREAD) i++;
|
|
237
|
+
resolvePromise(Math.max(0, Math.round(frames[i].t * 1000)));
|
|
238
|
+
});
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** A motionless span found by `freezedetect`, in ms from the file's start. */
|
|
243
|
+
export interface FreezeSpan {
|
|
244
|
+
startMs: number;
|
|
245
|
+
endMs: number;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Spans of `file` where nothing on screen moves for at least `minMs`
|
|
250
|
+
* (ffmpeg `freezedetect`). Used by compose's `autoIdle` to trim dead air the
|
|
251
|
+
* storyboard never annotated — a slow XHR, a fixed `pause`, a video the page
|
|
252
|
+
* hasn't started yet.
|
|
253
|
+
*
|
|
254
|
+
* Best-effort like `probeFlatTailMs`: any probe failure resolves to `[]`, so a
|
|
255
|
+
* missing filter or an odd container is a no-op, never a failed compose.
|
|
256
|
+
*/
|
|
257
|
+
export function probeFreezeSpans(
|
|
258
|
+
file: string,
|
|
259
|
+
minMs = 1500,
|
|
260
|
+
noise = 0.003
|
|
261
|
+
): Promise<FreezeSpan[]> {
|
|
262
|
+
const d = Math.max(0.1, minMs / 1000);
|
|
263
|
+
return new Promise((resolvePromise) => {
|
|
264
|
+
const proc = spawn(
|
|
265
|
+
"ffmpeg",
|
|
266
|
+
[
|
|
267
|
+
"-hide_banner",
|
|
268
|
+
"-nostats",
|
|
269
|
+
"-i",
|
|
270
|
+
file,
|
|
271
|
+
"-vf",
|
|
272
|
+
`freezedetect=n=${noise}:d=${d.toFixed(3)},metadata=print:file=-`,
|
|
273
|
+
"-an",
|
|
274
|
+
"-f",
|
|
275
|
+
"null",
|
|
276
|
+
"-",
|
|
277
|
+
],
|
|
278
|
+
{ stdio: ["ignore", "pipe", "ignore"] }
|
|
279
|
+
);
|
|
280
|
+
let out = "";
|
|
281
|
+
proc.stdout.on("data", (d2) => (out += d2.toString()));
|
|
282
|
+
proc.on("error", () => resolvePromise([]));
|
|
283
|
+
proc.on("close", () => {
|
|
284
|
+
// freezedetect emits, per detected span:
|
|
285
|
+
// lavfi.freezedetect.freeze_start=12.34
|
|
286
|
+
// lavfi.freezedetect.freeze_duration=2.5
|
|
287
|
+
// lavfi.freezedetect.freeze_end=14.84
|
|
288
|
+
// A span still frozen at EOF has a start with no end.
|
|
289
|
+
const spans: FreezeSpan[] = [];
|
|
290
|
+
let start: number | null = null;
|
|
291
|
+
for (const line of out.split("\n")) {
|
|
292
|
+
const s = /freeze_start=\s*([\d.]+)/.exec(line);
|
|
293
|
+
if (s) {
|
|
294
|
+
start = parseFloat(s[1]);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
const e = /freeze_end=\s*([\d.]+)/.exec(line);
|
|
298
|
+
if (e && start != null) {
|
|
299
|
+
spans.push({ startMs: Math.round(start * 1000), endMs: Math.round(parseFloat(e[1]) * 1000) });
|
|
300
|
+
start = null;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
if (start != null) spans.push({ startMs: Math.round(start * 1000), endMs: Number.MAX_SAFE_INTEGER });
|
|
304
|
+
resolvePromise(spans.filter((s) => s.endMs > s.startMs));
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
}
|