@tandryukha/aidemo 0.8.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +609 -45
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +70 -6
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +580 -22
- package/src/compose.ts +1006 -93
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +9 -3
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +156 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +159 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +629 -0
- package/src/inspect.ts +307 -0
- package/src/lint.ts +601 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +4 -1
- package/src/mcp/server.ts +403 -26
- package/src/player.ts +975 -32
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/recorder.ts +252 -15
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +643 -5
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/variants.ts
CHANGED
|
@@ -60,7 +60,7 @@ export async function loadVariants(path: string): Promise<Variant[]> {
|
|
|
60
60
|
|
|
61
61
|
export interface RenderVariantsOptions {
|
|
62
62
|
/** Chrome record options shared by every variant. */
|
|
63
|
-
record?: Pick<RecordOptions, "profileDir" | "headed" | "capture">;
|
|
63
|
+
record?: Pick<RecordOptions, "profileDir" | "fresh" | "headed" | "capture">;
|
|
64
64
|
/** Re-synthesize narration even if unchanged. */
|
|
65
65
|
forceVoice?: boolean;
|
|
66
66
|
}
|
package/src/voice-local.ts
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
ttsModelCacheDir,
|
|
14
14
|
} from "./config.js";
|
|
15
15
|
import { runFfmpeg } from "./ffmpeg.js";
|
|
16
|
-
import { exists, log } from "./util.js";
|
|
16
|
+
import { exists, log, sleep } from "./util.js";
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* In-process TTS: Kokoro-82M ONNX on CPU via kokoro-js — no server, no key.
|
|
@@ -211,6 +211,12 @@ export class LocalVoiceProvider implements VoiceProvider {
|
|
|
211
211
|
const tts = await this.tts.catch(() => null);
|
|
212
212
|
await tts?.model?.dispose().catch(() => {});
|
|
213
213
|
this.tts = null;
|
|
214
|
+
// The session release resolves before onnxruntime's native worker threads
|
|
215
|
+
// have fully wound down; a process exit that follows immediately (CLI end,
|
|
216
|
+
// MCP client close) can race that teardown and abort with a shutdown-time
|
|
217
|
+
// "libc++abi: mutex lock failed" on macOS (issue #19). Purely cosmetic —
|
|
218
|
+
// the run already succeeded — but a short grace keeps exits off the race.
|
|
219
|
+
await sleep(500);
|
|
214
220
|
}
|
|
215
221
|
}
|
|
216
222
|
|
package/src/voice.ts
CHANGED
|
@@ -119,14 +119,70 @@ function buildInstructions(plan: VoicePlan): string {
|
|
|
119
119
|
return parts.join(" ") || "Clear, friendly, natural narration.";
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Kokoro (local TTS) voice ids look like af_heart / am_adam / bf_emma — a
|
|
124
|
+
* two-letter language+gender prefix and an underscore. No OpenAI voice
|
|
125
|
+
* contains an underscore, so a storyboard authored for the local provider but
|
|
126
|
+
* run without AIDEMO_TTS_PROVIDER=local would otherwise reach api.openai.com
|
|
127
|
+
* and die mid-run on an opaque `400 Invalid value: 'af_heart'` (issue #22).
|
|
128
|
+
* Catch it up-front, before any scene is synthesized, with the actual fix.
|
|
129
|
+
* Custom OPENAI_BASE_URL servers are exempt: an OpenAI-compatible Kokoro
|
|
130
|
+
* server (Kokoro-FastAPI, speaches) legitimately serves these ids.
|
|
131
|
+
*/
|
|
132
|
+
const KOKORO_VOICE_ID_RE = /^[a-z]{2}_[a-z0-9_]+$/;
|
|
133
|
+
|
|
134
|
+
function assertVoiceIdsMatchProvider(storyboard: Storyboard): void {
|
|
135
|
+
if (openAiBaseUrl()) return;
|
|
136
|
+
const offender = storyboard.scenes
|
|
137
|
+
.map((s) => planFor(storyboard, s.voice).voiceId)
|
|
138
|
+
.find((v) => KOKORO_VOICE_ID_RE.test(v));
|
|
139
|
+
if (!offender) return;
|
|
140
|
+
throw new Error(
|
|
141
|
+
`voiceId "${offender}" is a Kokoro voice, but the active TTS provider is ` +
|
|
142
|
+
`OpenAI — the request would fail with "400 Invalid value: '${offender}'". ` +
|
|
143
|
+
`Run with AIDEMO_TTS_PROVIDER=local (needs: npm install kokoro-js) to ` +
|
|
144
|
+
`synthesize it in-process, or pick an OpenAI voice (marin, alloy, verse, …). ` +
|
|
145
|
+
`The provider is selected by the AIDEMO_TTS_PROVIDER env var (or the ` +
|
|
146
|
+
`--tts flag on voice/render), not by the storyboard.`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
122
150
|
function planFor(storyboard: Storyboard, sceneVoice?: VoicePlan): VoicePlan {
|
|
151
|
+
const pronounce =
|
|
152
|
+
storyboard.voice?.pronounce || sceneVoice?.pronounce
|
|
153
|
+
? { ...(storyboard.voice?.pronounce ?? {}), ...(sceneVoice?.pronounce ?? {}) }
|
|
154
|
+
: undefined;
|
|
123
155
|
return {
|
|
124
156
|
voiceId: sceneVoice?.voiceId ?? storyboard.voice?.voiceId ?? "marin",
|
|
125
157
|
instructions: sceneVoice?.instructions ?? storyboard.voice?.instructions,
|
|
126
158
|
speed: sceneVoice?.speed ?? storyboard.voice?.speed,
|
|
159
|
+
...(pronounce ? { pronounce } : {}),
|
|
127
160
|
};
|
|
128
161
|
}
|
|
129
162
|
|
|
163
|
+
/**
|
|
164
|
+
* Apply `voice.pronounce` to the text handed to TTS: whole-word, case-sensitive
|
|
165
|
+
* substitutions (longest key first so "SQL Server" wins over "SQL"). The
|
|
166
|
+
* narration itself — and so the script-timed captions — keeps the written
|
|
167
|
+
* form; only the spoken audio changes.
|
|
168
|
+
*/
|
|
169
|
+
export function applyPronounce(text: string, map?: Record<string, string>): string {
|
|
170
|
+
if (!map) return text;
|
|
171
|
+
const keys = Object.keys(map)
|
|
172
|
+
.filter((k) => k.trim())
|
|
173
|
+
.sort((a, b) => b.length - a.length);
|
|
174
|
+
let out = text;
|
|
175
|
+
for (const k of keys) {
|
|
176
|
+
const esc = k.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
177
|
+
// Word-ish boundaries that also work for keys with punctuation ("k8s", "C#").
|
|
178
|
+
out = out.replace(
|
|
179
|
+
new RegExp(`(^|[^\\p{L}\\p{N}_])${esc}(?=$|[^\\p{L}\\p{N}_])`, "gu"),
|
|
180
|
+
(_m, pre: string) => `${pre}${map[k]}`
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
return out;
|
|
184
|
+
}
|
|
185
|
+
|
|
130
186
|
/**
|
|
131
187
|
* Identity of a scene's audio = its narration + the resolved voice plan (+ the
|
|
132
188
|
* TTS provider when non-default; for the local provider that includes the
|
|
@@ -201,6 +257,7 @@ export async function generateVoice(
|
|
|
201
257
|
// The audio identity key: local audio also changes with the quantization.
|
|
202
258
|
const providerKey =
|
|
203
259
|
providerName === "local" ? `local:${kokoroDtype()}` : providerName;
|
|
260
|
+
if (providerName === "openai") assertVoiceIdsMatchProvider(storyboard);
|
|
204
261
|
if (providerName === "local") warnLocalLangCoverage(project.lang);
|
|
205
262
|
const base = openAiBaseUrl();
|
|
206
263
|
step(
|
|
@@ -276,7 +333,9 @@ export async function generateVoice(
|
|
|
276
333
|
scene.narration.length > 48 ? "…" : ""
|
|
277
334
|
}" (${plan.voiceId})`
|
|
278
335
|
);
|
|
279
|
-
const
|
|
336
|
+
const spoken = applyPronounce(scene.narration, plan.pronounce);
|
|
337
|
+
if (spoken !== scene.narration) log(` pronounce → "${spoken.slice(0, 64)}…"`);
|
|
338
|
+
const audio = await provider().synthesize({ text: spoken, plan });
|
|
280
339
|
await fs.writeFile(outPath, audio);
|
|
281
340
|
const durationMs = await probeDurationMs(outPath);
|
|
282
341
|
manifest.scenes.push({ id: scene.id, file: outPath, durationMs, hash });
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-artifact export from one take: a keyboard-navigable HTML walkthrough
|
|
3
|
+
* (one card per scene: payoff frame + narration + timestamp), a Markdown
|
|
4
|
+
* guide (README/SOP-ready), the SRT/VTT captions, and a JSON manifest — all
|
|
5
|
+
* derived from the FINAL video and compose's report.json, so the frames show
|
|
6
|
+
* exactly what the video shows (zoom, cursor, overlays, frame). No re-take,
|
|
7
|
+
* no re-encode: one `ffmpeg -ss` per scene.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { promises as fs } from "node:fs";
|
|
11
|
+
import { resolve, isAbsolute, basename } from "node:path";
|
|
12
|
+
import { Project } from "./project.js";
|
|
13
|
+
import { ComposeReportSchema, type Storyboard } from "./types.js";
|
|
14
|
+
import { runFfmpeg } from "./ffmpeg.js";
|
|
15
|
+
import { ensureDir, exists, readJson, writeJson, log, step } from "./util.js";
|
|
16
|
+
|
|
17
|
+
export interface WalkthroughOptions {
|
|
18
|
+
/** Output directory. Default <dir>/output/walkthrough (walkthrough.<lang> for variants). */
|
|
19
|
+
outDir?: string;
|
|
20
|
+
/** Frame width in px (aspect kept). Default 960. */
|
|
21
|
+
width?: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface WalkthroughScene {
|
|
25
|
+
index: number;
|
|
26
|
+
id: string;
|
|
27
|
+
title: string;
|
|
28
|
+
narration: string;
|
|
29
|
+
/** Content time in the final video, ms. */
|
|
30
|
+
startMs: number;
|
|
31
|
+
endMs: number;
|
|
32
|
+
/** Frame file (relative to the walkthrough dir). */
|
|
33
|
+
image: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface WalkthroughResult {
|
|
37
|
+
dir: string;
|
|
38
|
+
index: string;
|
|
39
|
+
guide: string;
|
|
40
|
+
manifest: string;
|
|
41
|
+
scenes: WalkthroughScene[];
|
|
42
|
+
captions: { srt?: string; vtt?: string };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const esc = (s: string): string =>
|
|
46
|
+
s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
47
|
+
|
|
48
|
+
const mmss = (ms: number): string => {
|
|
49
|
+
const s = Math.floor(ms / 1000);
|
|
50
|
+
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export async function exportWalkthrough(
|
|
54
|
+
project: Project,
|
|
55
|
+
storyboard: Storyboard,
|
|
56
|
+
opts: WalkthroughOptions = {}
|
|
57
|
+
): Promise<WalkthroughResult> {
|
|
58
|
+
const video = project.outputPath;
|
|
59
|
+
if (!(await exists(video))) {
|
|
60
|
+
throw new Error(`no video at ${video} — run 'aidemo render ${project.dir}' (or compose) first`);
|
|
61
|
+
}
|
|
62
|
+
if (!(await exists(project.reportPath))) {
|
|
63
|
+
throw new Error(`no report at ${project.reportPath} — re-run 'aidemo compose ${project.dir}'`);
|
|
64
|
+
}
|
|
65
|
+
const report = ComposeReportSchema.parse(await readJson(project.reportPath));
|
|
66
|
+
const width = Math.max(240, Math.round(opts.width ?? 960));
|
|
67
|
+
const dir = opts.outDir
|
|
68
|
+
? isAbsolute(opts.outDir)
|
|
69
|
+
? opts.outDir
|
|
70
|
+
: resolve(process.cwd(), opts.outDir)
|
|
71
|
+
: project.p("output", project.lang ? `walkthrough.${project.lang}` : "walkthrough");
|
|
72
|
+
await ensureDir(dir);
|
|
73
|
+
step(`Exporting walkthrough (${report.scenes.length} scene(s), ${width}px frames)`);
|
|
74
|
+
|
|
75
|
+
const byId = new Map(storyboard.scenes.map((s) => [s.id, s]));
|
|
76
|
+
const introMs = storyboard.intro?.durationMs ?? 0;
|
|
77
|
+
const scenes: WalkthroughScene[] = [];
|
|
78
|
+
let t = introMs;
|
|
79
|
+
for (let i = 0; i < report.scenes.length; i++) {
|
|
80
|
+
const r = report.scenes[i];
|
|
81
|
+
const sb = byId.get(r.id);
|
|
82
|
+
const startMs = t;
|
|
83
|
+
const endMs = t + r.targetMs;
|
|
84
|
+
t = endMs;
|
|
85
|
+
// Payoff frame: near the end of the scene (actions done, overlays and
|
|
86
|
+
// holds visible), never the first frames (which still show the previous
|
|
87
|
+
// screen while a navigation lands).
|
|
88
|
+
const seekMs = Math.max(startMs, endMs - Math.min(600, Math.max(100, r.targetMs * 0.25)));
|
|
89
|
+
const image = `scene-${String(i + 1).padStart(2, "0")}-${r.id.replace(/[^\w.-]+/g, "_")}.png`;
|
|
90
|
+
await runFfmpeg([
|
|
91
|
+
"-ss",
|
|
92
|
+
(seekMs / 1000).toFixed(3),
|
|
93
|
+
"-i",
|
|
94
|
+
video,
|
|
95
|
+
"-frames:v",
|
|
96
|
+
"1",
|
|
97
|
+
"-vf",
|
|
98
|
+
`scale=${width}:-2`,
|
|
99
|
+
"-update",
|
|
100
|
+
"1",
|
|
101
|
+
resolve(dir, image),
|
|
102
|
+
]);
|
|
103
|
+
scenes.push({
|
|
104
|
+
index: i + 1,
|
|
105
|
+
id: r.id,
|
|
106
|
+
title: sb?.title ?? r.id,
|
|
107
|
+
narration: sb?.narration ?? "",
|
|
108
|
+
startMs,
|
|
109
|
+
endMs,
|
|
110
|
+
image,
|
|
111
|
+
});
|
|
112
|
+
log(`scene ${r.id}: ${mmss(startMs)} → ${image}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Captions travel with the walkthrough (players and LMSs want the files).
|
|
116
|
+
const captions: { srt?: string; vtt?: string } = {};
|
|
117
|
+
for (const [key, src] of [
|
|
118
|
+
["srt", project.captionsSrtPath],
|
|
119
|
+
["vtt", project.captionsVttPath],
|
|
120
|
+
] as const) {
|
|
121
|
+
if (await exists(src)) {
|
|
122
|
+
const dst = resolve(dir, basename(src));
|
|
123
|
+
await fs.copyFile(src, dst);
|
|
124
|
+
captions[key] = dst;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const videoRel = `../${basename(video)}`;
|
|
129
|
+
const title = storyboard.title;
|
|
130
|
+
const guide = resolve(dir, "guide.md");
|
|
131
|
+
await fs.writeFile(
|
|
132
|
+
guide,
|
|
133
|
+
[
|
|
134
|
+
`# ${title}`,
|
|
135
|
+
"",
|
|
136
|
+
`Video: [${basename(video)}](${videoRel}) · ${mmss(report.durationMs)}`,
|
|
137
|
+
"",
|
|
138
|
+
...scenes.flatMap((s) => [
|
|
139
|
+
`## ${s.index}. ${s.title}`,
|
|
140
|
+
"",
|
|
141
|
+
``,
|
|
142
|
+
"",
|
|
143
|
+
s.narration,
|
|
144
|
+
"",
|
|
145
|
+
`_${mmss(s.startMs)}–${mmss(s.endMs)}_`,
|
|
146
|
+
"",
|
|
147
|
+
]),
|
|
148
|
+
].join("\n")
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
const index = resolve(dir, "index.html");
|
|
152
|
+
await fs.writeFile(index, renderHtml(title, videoRel, scenes, report.durationMs));
|
|
153
|
+
const manifest = resolve(dir, "walkthrough.json");
|
|
154
|
+
await writeJson(manifest, {
|
|
155
|
+
title,
|
|
156
|
+
video: videoRel,
|
|
157
|
+
durationMs: report.durationMs,
|
|
158
|
+
scenes,
|
|
159
|
+
captions: Object.fromEntries(Object.entries(captions).map(([k, v]) => [k, basename(v)])),
|
|
160
|
+
});
|
|
161
|
+
log(`walkthrough → ${index}`);
|
|
162
|
+
return { dir, index, guide, manifest, scenes, captions };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function renderHtml(
|
|
166
|
+
title: string,
|
|
167
|
+
videoRel: string,
|
|
168
|
+
scenes: WalkthroughScene[],
|
|
169
|
+
durationMs: number
|
|
170
|
+
): string {
|
|
171
|
+
const cards = scenes
|
|
172
|
+
.map(
|
|
173
|
+
(s) => `
|
|
174
|
+
<section class="scene" id="s${s.index}" data-start="${(s.startMs / 1000).toFixed(2)}">
|
|
175
|
+
<img src="${esc(s.image)}" alt="${esc(s.title)}" loading="lazy">
|
|
176
|
+
<div class="meta">
|
|
177
|
+
<div class="step">${s.index} / ${scenes.length}</div>
|
|
178
|
+
<h2>${esc(s.title)}</h2>
|
|
179
|
+
<p>${esc(s.narration)}</p>
|
|
180
|
+
<button class="jump" type="button">▶ ${mmss(s.startMs)}</button>
|
|
181
|
+
</div>
|
|
182
|
+
</section>`
|
|
183
|
+
)
|
|
184
|
+
.join("\n");
|
|
185
|
+
return `<!doctype html>
|
|
186
|
+
<html lang="en">
|
|
187
|
+
<head>
|
|
188
|
+
<meta charset="utf-8">
|
|
189
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
190
|
+
<title>${esc(title)} — walkthrough</title>
|
|
191
|
+
<style>
|
|
192
|
+
:root { color-scheme: light dark; --fg: #111; --muted: #666; --bg: #fff; --card: #f6f7f9; --accent: #3b82f6; }
|
|
193
|
+
@media (prefers-color-scheme: dark) { :root { --fg: #eef; --muted: #aab; --bg: #0e1118; --card: #171b26; } }
|
|
194
|
+
* { box-sizing: border-box; }
|
|
195
|
+
body { margin: 0; background: var(--bg); color: var(--fg); font: 16px/1.5 -apple-system, "Segoe UI", Helvetica, Arial, sans-serif; }
|
|
196
|
+
header { padding: 24px 24px 8px; max-width: 1100px; margin: 0 auto; }
|
|
197
|
+
header h1 { margin: 0 0 4px; font-size: 24px; }
|
|
198
|
+
header .hint { color: var(--muted); font-size: 13px; }
|
|
199
|
+
video { display: block; width: 100%; max-width: 1100px; margin: 12px auto 24px; border-radius: 10px; background: #000; }
|
|
200
|
+
main { max-width: 1100px; margin: 0 auto; padding: 0 24px 48px; display: grid; gap: 20px; }
|
|
201
|
+
.scene { display: grid; grid-template-columns: minmax(0, 3fr) minmax(0, 2fr); gap: 20px; background: var(--card); border-radius: 12px; padding: 14px; scroll-margin-top: 16px; outline: 2px solid transparent; }
|
|
202
|
+
.scene.active { outline-color: var(--accent); }
|
|
203
|
+
.scene img { width: 100%; height: auto; border-radius: 8px; display: block; }
|
|
204
|
+
.meta { display: flex; flex-direction: column; gap: 6px; }
|
|
205
|
+
.step { color: var(--muted); font-size: 12px; letter-spacing: .06em; text-transform: uppercase; }
|
|
206
|
+
h2 { margin: 0; font-size: 18px; }
|
|
207
|
+
p { margin: 0; }
|
|
208
|
+
.jump { align-self: flex-start; margin-top: auto; border: 1px solid var(--accent); color: var(--accent); background: transparent; border-radius: 6px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px; }
|
|
209
|
+
@media (max-width: 720px) { .scene { grid-template-columns: 1fr; } }
|
|
210
|
+
</style>
|
|
211
|
+
</head>
|
|
212
|
+
<body>
|
|
213
|
+
<header>
|
|
214
|
+
<h1>${esc(title)}</h1>
|
|
215
|
+
<div class="hint">${scenes.length} steps · ${mmss(durationMs)} · ← → to move between steps, Enter to play from a step</div>
|
|
216
|
+
</header>
|
|
217
|
+
<video id="v" src="${esc(videoRel)}" controls preload="metadata"></video>
|
|
218
|
+
<main>${cards}
|
|
219
|
+
</main>
|
|
220
|
+
<script>
|
|
221
|
+
(() => {
|
|
222
|
+
const v = document.getElementById("v");
|
|
223
|
+
const scenes = Array.from(document.querySelectorAll(".scene"));
|
|
224
|
+
let cur = 0;
|
|
225
|
+
const activate = (i, scroll) => {
|
|
226
|
+
cur = Math.max(0, Math.min(scenes.length - 1, i));
|
|
227
|
+
scenes.forEach((s, k) => s.classList.toggle("active", k === cur));
|
|
228
|
+
if (scroll) scenes[cur].scrollIntoView({ behavior: "smooth", block: "start" });
|
|
229
|
+
};
|
|
230
|
+
const play = (i) => { activate(i, false); v.currentTime = Number(scenes[cur].dataset.start || 0); v.play().catch(() => {}); v.scrollIntoView({ behavior: "smooth", block: "start" }); };
|
|
231
|
+
scenes.forEach((s, i) => { s.addEventListener("click", () => activate(i, false)); s.querySelector(".jump").addEventListener("click", (e) => { e.stopPropagation(); play(i); }); });
|
|
232
|
+
document.addEventListener("keydown", (e) => {
|
|
233
|
+
if (e.target && /input|textarea/i.test(e.target.tagName)) return;
|
|
234
|
+
if (e.key === "ArrowRight" || e.key === "j") { activate(cur + 1, true); e.preventDefault(); }
|
|
235
|
+
else if (e.key === "ArrowLeft" || e.key === "k") { activate(cur - 1, true); e.preventDefault(); }
|
|
236
|
+
else if (e.key === "Enter") { play(cur); e.preventDefault(); }
|
|
237
|
+
});
|
|
238
|
+
v.addEventListener("timeupdate", () => {
|
|
239
|
+
const t = v.currentTime;
|
|
240
|
+
let i = 0;
|
|
241
|
+
scenes.forEach((s, k) => { if (t >= Number(s.dataset.start || 0)) i = k; });
|
|
242
|
+
if (i !== cur) activate(i, false);
|
|
243
|
+
});
|
|
244
|
+
activate(0, false);
|
|
245
|
+
})();
|
|
246
|
+
</script>
|
|
247
|
+
</body>
|
|
248
|
+
</html>
|
|
249
|
+
`;
|
|
250
|
+
}
|
package/src/zoom.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ZoomConfig } from "./types.js";
|
|
2
|
+
import { type Key, piecewiseExpr } from "./expr.js";
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Screen-Studio-style auto-zoom, rendered at compose time with ffmpeg's
|
|
@@ -12,9 +13,37 @@ import type { ZoomConfig } from "./types.js";
|
|
|
12
13
|
* and in again — the camera stays zoomed and *pans* between them.
|
|
13
14
|
*
|
|
14
15
|
* Everything is emitted as piecewise smoothstep ffmpeg expressions over output
|
|
15
|
-
* time (`on/fps`), one channel each for zoom, center-x and center-y
|
|
16
|
+
* time (`on/fps`), one channel each for zoom, center-x and center-y — as a FLAT
|
|
17
|
+
* gated sum, and simplified to a key budget, because a long take's worth of
|
|
18
|
+
* focus points overflows ffmpeg's expression parser otherwise (see expr.ts).
|
|
16
19
|
*/
|
|
17
20
|
|
|
21
|
+
/**
|
|
22
|
+
* Per-channel key budget. zoom/x/y are three INDEPENDENT expressions, so this
|
|
23
|
+
* is per channel; it is deliberately well under ffmpeg's ~100-term AST cliff
|
|
24
|
+
* because each term references the time expression three times.
|
|
25
|
+
*/
|
|
26
|
+
const KEY_BUDGET = 56;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Smallest slice of the page, in LOGICAL (storyboard) pixels, that a zoom is
|
|
30
|
+
* allowed to leave visible horizontally. A scale tuned on a 1440px desktop
|
|
31
|
+
* viewport crops a 430px mobile take to ribbons — 1.35x cost the question
|
|
32
|
+
* headline its left edge on every click (issue #38). Anything at or below this
|
|
33
|
+
* width stops zooming further in.
|
|
34
|
+
*/
|
|
35
|
+
const MIN_VISIBLE_LOGICAL_WIDTH = 360;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Cap `scale` so a zoom never leaves less than MIN_VISIBLE_LOGICAL_WIDTH of the
|
|
39
|
+
* page in frame. Returns the configured scale unchanged on desktop-width takes.
|
|
40
|
+
*/
|
|
41
|
+
export function clampScaleForWidth(scale: number, logicalWidth: number): number {
|
|
42
|
+
if (logicalWidth <= 0) return scale;
|
|
43
|
+
const max = Math.max(1.05, logicalWidth / MIN_VISIBLE_LOGICAL_WIDTH);
|
|
44
|
+
return Math.min(scale, max);
|
|
45
|
+
}
|
|
46
|
+
|
|
18
47
|
/** A focus point in FINAL video time, coordinates in output-video pixels. */
|
|
19
48
|
export interface ZoomEvent {
|
|
20
49
|
tMs: number;
|
|
@@ -24,11 +53,6 @@ export interface ZoomEvent {
|
|
|
24
53
|
holdMs?: number;
|
|
25
54
|
}
|
|
26
55
|
|
|
27
|
-
interface Key {
|
|
28
|
-
t: number; // seconds
|
|
29
|
-
v: number;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
56
|
/** Append a keyframe, nudging time forward to keep the channel monotonic. */
|
|
33
57
|
function pushKey(keys: Key[], t: number, v: number): void {
|
|
34
58
|
const last = keys[keys.length - 1];
|
|
@@ -36,34 +60,6 @@ function pushKey(keys: Key[], t: number, v: number): void {
|
|
|
36
60
|
keys.push({ t, v });
|
|
37
61
|
}
|
|
38
62
|
|
|
39
|
-
const num = (n: number): string => {
|
|
40
|
-
const s = n.toFixed(4).replace(/\.?0+$/, "");
|
|
41
|
-
return s === "" || s === "-" ? "0" : s;
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Piecewise expression over `T` (a time expression in seconds): holds the
|
|
46
|
-
* first value before the first key, smoothsteps between keys, holds the last
|
|
47
|
-
* value after the last key.
|
|
48
|
-
*/
|
|
49
|
-
function piecewise(keys: Key[], T: string): string {
|
|
50
|
-
let expr = num(keys[keys.length - 1].v);
|
|
51
|
-
for (let i = keys.length - 2; i >= 0; i--) {
|
|
52
|
-
const a = keys[i];
|
|
53
|
-
const b = keys[i + 1];
|
|
54
|
-
let seg: string;
|
|
55
|
-
if (a.v === b.v || b.t - a.t < 1e-6) {
|
|
56
|
-
seg = num(a.v);
|
|
57
|
-
} else {
|
|
58
|
-
const p = `((${T}-${num(a.t)})/${num(b.t - a.t)})`;
|
|
59
|
-
seg = `(${num(a.v)}+(${num(b.v - a.v)})*${p}*${p}*(3-2*${p}))`;
|
|
60
|
-
}
|
|
61
|
-
expr = `if(lt(${T},${num(b.t)}),${seg},${expr})`;
|
|
62
|
-
}
|
|
63
|
-
// Guard the span before the first key (p would be negative in seg 0).
|
|
64
|
-
return `if(lt(${T},${num(keys[0].t)}),${num(keys[0].v)},${expr})`;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
63
|
/**
|
|
68
64
|
* Build the zoom/pan keyframes for all three channels.
|
|
69
65
|
* Exported for tests/debugging; buildZoomFilter is the real entry point.
|
|
@@ -147,6 +143,42 @@ export function planZoom(
|
|
|
147
143
|
return { z, cx, cy };
|
|
148
144
|
}
|
|
149
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Plan the zoom, thinning focus events until every channel fits KEY_BUDGET.
|
|
148
|
+
*
|
|
149
|
+
* A long take can carry forty-plus focus points, and each one costs a handful
|
|
150
|
+
* of keys per channel — more than ffmpeg's expression evaluator will take
|
|
151
|
+
* (see expr.ts). Simplifying the CURVE is the wrong lever here: the zoom
|
|
152
|
+
* channel is a train of bounces, and smoothing it enough to fit flattens the
|
|
153
|
+
* zoom out of existence. Dropping whole focus points keeps every surviving
|
|
154
|
+
* beat at full amplitude, evenly spread across the demo — the same trade the
|
|
155
|
+
* author would make by hand, and it is logged so it is not a surprise.
|
|
156
|
+
*/
|
|
157
|
+
function planWithinBudget(
|
|
158
|
+
events: ZoomEvent[],
|
|
159
|
+
cfg: ZoomConfig,
|
|
160
|
+
outW: number,
|
|
161
|
+
outH: number,
|
|
162
|
+
durMs: number
|
|
163
|
+
): { plan: ReturnType<typeof planZoom>; dropped: number } {
|
|
164
|
+
let use = events;
|
|
165
|
+
for (let i = 0; i < 24; i++) {
|
|
166
|
+
const plan = planZoom(use, cfg, outW, outH, durMs);
|
|
167
|
+
if (!plan) return { plan, dropped: 0 };
|
|
168
|
+
const worst = Math.max(plan.z.length, plan.cx.length, plan.cy.length);
|
|
169
|
+
if (worst <= KEY_BUDGET || use.length <= 2) {
|
|
170
|
+
return { plan, dropped: events.length - use.length };
|
|
171
|
+
}
|
|
172
|
+
// Thin proportionally to the overshoot, keeping an even spread.
|
|
173
|
+
const keep = Math.max(2, Math.floor((use.length * KEY_BUDGET) / worst));
|
|
174
|
+
const step = use.length / keep;
|
|
175
|
+
const next: ZoomEvent[] = [];
|
|
176
|
+
for (let k = 0; k < keep; k++) next.push(use[Math.floor(k * step)]);
|
|
177
|
+
use = next;
|
|
178
|
+
}
|
|
179
|
+
return { plan: planZoom(use, cfg, outW, outH, durMs), dropped: events.length - use.length };
|
|
180
|
+
}
|
|
181
|
+
|
|
150
182
|
/**
|
|
151
183
|
* Full `-vf` filter string applying the auto-zoom to a CFR video of
|
|
152
184
|
* outW x outH at `fps`, or null when there is nothing to zoom. For SD-ish
|
|
@@ -159,18 +191,32 @@ export function buildZoomFilter(
|
|
|
159
191
|
outW: number,
|
|
160
192
|
outH: number,
|
|
161
193
|
durMs: number,
|
|
162
|
-
fps: number
|
|
194
|
+
fps: number,
|
|
195
|
+
/** Storyboard (logical) viewport width, for the narrow-viewport scale cap. */
|
|
196
|
+
logicalWidth = outW,
|
|
197
|
+
/** Called when focus events had to be thinned to fit the key budget. */
|
|
198
|
+
onThin?: (dropped: number, total: number) => void
|
|
163
199
|
): string | null {
|
|
164
|
-
const
|
|
200
|
+
const capped: ZoomConfig = {
|
|
201
|
+
...cfg,
|
|
202
|
+
scale: clampScaleForWidth(cfg.scale, logicalWidth),
|
|
203
|
+
};
|
|
204
|
+
const clamped = events.map((e) =>
|
|
205
|
+
e.scale == null
|
|
206
|
+
? e
|
|
207
|
+
: { ...e, scale: clampScaleForWidth(e.scale, logicalWidth) }
|
|
208
|
+
);
|
|
209
|
+
const { plan, dropped } = planWithinBudget(clamped, capped, outW, outH, durMs);
|
|
165
210
|
if (!plan) return null;
|
|
211
|
+
if (dropped > 0) onThin?.(dropped, clamped.length);
|
|
166
212
|
|
|
167
213
|
const up = outW < 1600 ? 2 : 1;
|
|
168
214
|
const T = `on/${fps}`;
|
|
169
|
-
const zExpr = piecewise(plan.z, T);
|
|
170
215
|
const scaleKeys = (keys: Key[]): Key[] =>
|
|
171
216
|
keys.map((k) => ({ t: k.t, v: k.v * up }));
|
|
172
|
-
const
|
|
173
|
-
const
|
|
217
|
+
const zExpr = piecewiseExpr(plan.z, T, "smooth");
|
|
218
|
+
const cxExpr = piecewiseExpr(scaleKeys(plan.cx), T, "smooth");
|
|
219
|
+
const cyExpr = piecewiseExpr(scaleKeys(plan.cy), T, "smooth");
|
|
174
220
|
|
|
175
221
|
const xExpr = `clip(${cxExpr}-(iw/zoom)/2,0,iw-iw/zoom)`;
|
|
176
222
|
const yExpr = `clip(${cyExpr}-(ih/zoom)/2,0,ih-ih/zoom)`;
|