@cosmicdrift/kumiko-testing 0.416.0 → 0.417.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/changes.json +12 -0
- package/src/e2e/index.ts +11 -0
- package/src/e2e/screenshots.ts +4 -1
- package/src/e2e/video/cursor-overlay.ts +137 -0
- package/src/e2e/video/define-video.ts +79 -0
- package/src/e2e/video/frame-timeline.ts +141 -0
- package/src/e2e/video/mux.ts +185 -0
- package/src/e2e/video/run-videos.ts +179 -0
- package/src/e2e/video/tts.ts +95 -0
- package/src/e2e/video/video-texts.ts +47 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-testing",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.417.0",
|
|
4
4
|
"description": "Test template for Kumiko apps: per-flow seedTenant, app test stack, canonical test preloads, bunfig generator and integration runner.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -107,9 +107,9 @@
|
|
|
107
107
|
"kumiko-testing": "./bin/kumiko-testing.ts"
|
|
108
108
|
},
|
|
109
109
|
"dependencies": {
|
|
110
|
-
"@cosmicdrift/kumiko-framework": "0.
|
|
111
|
-
"@cosmicdrift/kumiko-guards": "0.
|
|
112
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
110
|
+
"@cosmicdrift/kumiko-framework": "0.417.0",
|
|
111
|
+
"@cosmicdrift/kumiko-guards": "0.417.0",
|
|
112
|
+
"@cosmicdrift/kumiko-types": "0.417.0",
|
|
113
113
|
"zod": "^4.6.7"
|
|
114
114
|
},
|
|
115
115
|
"peerDependencies": {
|
|
@@ -133,8 +133,8 @@
|
|
|
133
133
|
}
|
|
134
134
|
},
|
|
135
135
|
"devDependencies": {
|
|
136
|
-
"@cosmicdrift/kumiko-bundled-features": "0.
|
|
137
|
-
"@cosmicdrift/kumiko-dev-server": "0.
|
|
136
|
+
"@cosmicdrift/kumiko-bundled-features": "0.417.0",
|
|
137
|
+
"@cosmicdrift/kumiko-dev-server": "0.417.0",
|
|
138
138
|
"@happy-dom/global-registrator": "^21.0.4",
|
|
139
139
|
"@playwright/test": "^1.64.1",
|
|
140
140
|
"@testing-library/dom": "^10.4.2",
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.417.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "Video generator for tutorial and marketing videos",
|
|
6
|
+
"detail": "`defineVideo` and `runVideos` from `@cosmicdrift/kumiko-testing/e2e` record scenes from the real app or from an HTML stage with Playwright in virtual time, narrate them via Gemini TTS over OpenRouter (cached by text hash), and mux MP4 plus VTT with ffmpeg. Optional brand logo, background music with ducking and a per-locale AI disclosure."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.417.0",
|
|
10
|
+
"type": "improvement",
|
|
11
|
+
"title": "Smaller web-ready video files",
|
|
12
|
+
"detail": "The video muxer encodes with CRF 28, preset slow and +faststart, so trailers shrink by about half and start playing while they load."
|
|
13
|
+
},
|
|
2
14
|
{
|
|
3
15
|
"version": "0.415.0",
|
|
4
16
|
"type": "fix",
|
package/src/e2e/index.ts
CHANGED
|
@@ -88,3 +88,14 @@ export {
|
|
|
88
88
|
type SeedTenantFixture,
|
|
89
89
|
test,
|
|
90
90
|
} from "./seeded-tenant-fixture";
|
|
91
|
+
export {
|
|
92
|
+
DEFAULT_TTS_MODEL,
|
|
93
|
+
defineVideo,
|
|
94
|
+
type VideoDefinition,
|
|
95
|
+
type VideoScene,
|
|
96
|
+
type VideoSceneContext,
|
|
97
|
+
type VideoTools,
|
|
98
|
+
type VideoVoice,
|
|
99
|
+
} from "./video/define-video";
|
|
100
|
+
export { type RunVideosOptions, runVideos } from "./video/run-videos";
|
|
101
|
+
export { findVideoTextGaps, loadVideoTexts } from "./video/video-texts";
|
package/src/e2e/screenshots.ts
CHANGED
|
@@ -144,7 +144,10 @@ function pageFingerprint(): string {
|
|
|
144
144
|
|
|
145
145
|
// Replaces fixed sleeps: settled = no data request in flight and an unchanged
|
|
146
146
|
// DOM/scroll-size/animation fingerprint over STABLE_POLLS consecutive polls.
|
|
147
|
-
async function waitForSettledPage(
|
|
147
|
+
export async function waitForSettledPage(
|
|
148
|
+
page: Page,
|
|
149
|
+
inFlightDataRequests: () => number,
|
|
150
|
+
): Promise<void> {
|
|
148
151
|
let previous: string | undefined;
|
|
149
152
|
let stablePolls = 0;
|
|
150
153
|
await expect
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import type { Page } from "@playwright/test";
|
|
2
|
+
|
|
3
|
+
export interface FocusBox {
|
|
4
|
+
readonly x: number;
|
|
5
|
+
readonly y: number;
|
|
6
|
+
readonly width: number;
|
|
7
|
+
readonly height: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface VideoOverlayApi {
|
|
11
|
+
move(x: number, y: number): void;
|
|
12
|
+
ripple(x: number, y: number): void;
|
|
13
|
+
clearRipples(): void;
|
|
14
|
+
focus(box: FocusBox | null): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
declare global {
|
|
18
|
+
interface Window {
|
|
19
|
+
__kumikoVideoOverlay?: VideoOverlayApi;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Runs in the browser, so it may not reference module scope.
|
|
24
|
+
export function installVideoOverlay(): void {
|
|
25
|
+
const POSITION_KEY = "kumiko:video-cursor";
|
|
26
|
+
const ACCENT = "#6366f1";
|
|
27
|
+
const SVG_NS = "http://www.w3.org/2000/svg";
|
|
28
|
+
const TOP = "2147483647";
|
|
29
|
+
|
|
30
|
+
const layer = document.createElement("div");
|
|
31
|
+
layer.style.cssText = `position:fixed;inset:0;pointer-events:none;z-index:${TOP};`;
|
|
32
|
+
|
|
33
|
+
const cursor = document.createElementNS(SVG_NS, "svg");
|
|
34
|
+
cursor.setAttribute("width", "24");
|
|
35
|
+
cursor.setAttribute("height", "24");
|
|
36
|
+
cursor.setAttribute("viewBox", "0 0 24 24");
|
|
37
|
+
cursor.style.cssText =
|
|
38
|
+
"position:absolute;left:0;top:0;pointer-events:none;filter:drop-shadow(0 2px 3px rgba(0,0,0,0.45));";
|
|
39
|
+
const arrow = document.createElementNS(SVG_NS, "path");
|
|
40
|
+
arrow.setAttribute("d", "M3 2 L3 19 L7.5 14.8 L10.6 21.5 L13.2 20.3 L10.1 13.7 L16.5 13.7 Z");
|
|
41
|
+
arrow.setAttribute("fill", "#ffffff");
|
|
42
|
+
arrow.setAttribute("stroke", "#111111");
|
|
43
|
+
arrow.setAttribute("stroke-width", "1.4");
|
|
44
|
+
arrow.setAttribute("stroke-linejoin", "round");
|
|
45
|
+
cursor.appendChild(arrow);
|
|
46
|
+
|
|
47
|
+
const focusFrame = document.createElement("div");
|
|
48
|
+
focusFrame.style.cssText = `position:absolute;display:none;pointer-events:none;box-sizing:border-box;border:3px solid ${ACCENT};border-radius:10px;box-shadow:0 0 0 4px ${ACCENT}33,0 0 24px ${ACCENT}88;`;
|
|
49
|
+
|
|
50
|
+
layer.append(focusFrame, cursor);
|
|
51
|
+
|
|
52
|
+
// Init scripts run before the document has elements, so the layer is mounted on first use.
|
|
53
|
+
const mount = (): void => {
|
|
54
|
+
if (!layer.isConnected) document.body.appendChild(layer);
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const placeCursor = (x: number, y: number): void => {
|
|
58
|
+
mount();
|
|
59
|
+
cursor.style.transform = `translate(${x}px, ${y}px)`;
|
|
60
|
+
try {
|
|
61
|
+
sessionStorage.setItem(POSITION_KEY, JSON.stringify({ x, y }));
|
|
62
|
+
} catch {
|
|
63
|
+
// Storage may be blocked; the cursor then restarts at the origin.
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
const api: VideoOverlayApi = {
|
|
68
|
+
move: placeCursor,
|
|
69
|
+
ripple: (x, y) => {
|
|
70
|
+
mount();
|
|
71
|
+
const ripple = document.createElement("div");
|
|
72
|
+
ripple.style.cssText = `position:absolute;left:${x - 22}px;top:${y - 22}px;width:44px;height:44px;border-radius:50%;pointer-events:none;background:${ACCENT}55;border:3px solid ${ACCENT};`;
|
|
73
|
+
ripple.dataset["kumikoRipple"] = "1";
|
|
74
|
+
layer.appendChild(ripple);
|
|
75
|
+
},
|
|
76
|
+
clearRipples: () => {
|
|
77
|
+
for (const ripple of layer.querySelectorAll("[data-kumiko-ripple]")) ripple.remove();
|
|
78
|
+
},
|
|
79
|
+
focus: (box) => {
|
|
80
|
+
mount();
|
|
81
|
+
if (box === null) {
|
|
82
|
+
focusFrame.style.display = "none";
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const pad = 6;
|
|
86
|
+
focusFrame.style.display = "block";
|
|
87
|
+
focusFrame.style.left = `${box.x - pad}px`;
|
|
88
|
+
focusFrame.style.top = `${box.y - pad}px`;
|
|
89
|
+
focusFrame.style.width = `${box.width + pad * 2}px`;
|
|
90
|
+
focusFrame.style.height = `${box.height + pad * 2}px`;
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
window.__kumikoVideoOverlay = api;
|
|
94
|
+
|
|
95
|
+
let restored = { x: 0, y: 0 };
|
|
96
|
+
try {
|
|
97
|
+
const raw = sessionStorage.getItem(POSITION_KEY);
|
|
98
|
+
if (raw !== null) {
|
|
99
|
+
const parsed: unknown = JSON.parse(raw);
|
|
100
|
+
if (
|
|
101
|
+
typeof parsed === "object" &&
|
|
102
|
+
parsed !== null &&
|
|
103
|
+
"x" in parsed &&
|
|
104
|
+
"y" in parsed &&
|
|
105
|
+
typeof parsed.x === "number" &&
|
|
106
|
+
typeof parsed.y === "number"
|
|
107
|
+
) {
|
|
108
|
+
restored = { x: parsed.x, y: parsed.y };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
} catch {
|
|
112
|
+
// Unreadable stored position: start at the origin.
|
|
113
|
+
}
|
|
114
|
+
cursor.style.transform = `translate(${restored.x}px, ${restored.y}px)`;
|
|
115
|
+
// A page without interaction still shows the cursor where the last page left it.
|
|
116
|
+
document.addEventListener("DOMContentLoaded", mount);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function addVideoOverlay(page: Page): Promise<void> {
|
|
120
|
+
await page.addInitScript(installVideoOverlay);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export async function moveCursor(page: Page, x: number, y: number): Promise<void> {
|
|
124
|
+
await page.evaluate(([cx, cy]) => window.__kumikoVideoOverlay?.move(cx, cy), [x, y] as const);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function rippleAt(page: Page, x: number, y: number): Promise<void> {
|
|
128
|
+
await page.evaluate(([cx, cy]) => window.__kumikoVideoOverlay?.ripple(cx, cy), [x, y] as const);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export async function clearRipples(page: Page): Promise<void> {
|
|
132
|
+
await page.evaluate(() => window.__kumikoVideoOverlay?.clearRipples());
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export async function showFocus(page: Page, box: FocusBox | null): Promise<void> {
|
|
136
|
+
await page.evaluate((target) => window.__kumikoVideoOverlay?.focus(target), box);
|
|
137
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { Locator, Page } from "@playwright/test";
|
|
2
|
+
import type { PresentIdentity } from "../present-identities";
|
|
3
|
+
import type { SeedTenantFixture } from "../seeded-tenant-fixture";
|
|
4
|
+
|
|
5
|
+
export interface VideoVoice {
|
|
6
|
+
readonly voice: string;
|
|
7
|
+
readonly instructions?: string;
|
|
8
|
+
readonly model?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export const DEFAULT_TTS_MODEL = "google/gemini-3.8-flash-tts";
|
|
12
|
+
|
|
13
|
+
export interface VideoTools {
|
|
14
|
+
moveTo(target: Locator): Promise<void>;
|
|
15
|
+
click(target: Locator): Promise<void>;
|
|
16
|
+
type(target: Locator, text: string): Promise<void>;
|
|
17
|
+
hold(ms: number): Promise<void>;
|
|
18
|
+
// Frames the element until the scene ends; call it after the scene has navigated.
|
|
19
|
+
focus(target: Locator): Promise<void>;
|
|
20
|
+
// PNG data URL of the current app view, with presented identities applied.
|
|
21
|
+
snapshot(): Promise<string>;
|
|
22
|
+
// Replaces the page with a self-contained HTML stage; its CSS animations start at 0 ms.
|
|
23
|
+
stage(html: string): Promise<void>;
|
|
24
|
+
// Steps the stage's CSS animations frame by frame, so the video time is exact.
|
|
25
|
+
animate(ms: number): Promise<void>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface VideoSceneContext {
|
|
29
|
+
readonly seedTenant: SeedTenantFixture;
|
|
30
|
+
readonly locale: string;
|
|
31
|
+
// Same contract as ScenarioFixtures.presentIdentities: replaces seeded addresses in every frame.
|
|
32
|
+
readonly presentIdentities: (mappings: readonly PresentIdentity[]) => void;
|
|
33
|
+
readonly tools: VideoTools;
|
|
34
|
+
// Ids created in `setup` that scenes need; scenes of one video share it.
|
|
35
|
+
readonly state: Map<string, unknown>;
|
|
36
|
+
// Length of the current scene's narration; 0 in `setup`. The scene holds at least this long.
|
|
37
|
+
readonly speechMs: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface VideoScene {
|
|
41
|
+
readonly id: string;
|
|
42
|
+
readonly do: (page: Page, ctx: VideoSceneContext) => Promise<void>;
|
|
43
|
+
readonly instructions?: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface VideoMusic {
|
|
47
|
+
// Absolute path to an mp3, wav or m4a; loops under the narration and ducks while it speaks.
|
|
48
|
+
readonly file: string;
|
|
49
|
+
// Default 0.35.
|
|
50
|
+
readonly volume?: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface VideoDefinition {
|
|
54
|
+
readonly music?: VideoMusic;
|
|
55
|
+
readonly voice: VideoVoice;
|
|
56
|
+
readonly viewport?: { readonly width: number; readonly height: number };
|
|
57
|
+
// Login/seed work; runs before recording starts.
|
|
58
|
+
readonly setup?: (page: Page, ctx: VideoSceneContext) => Promise<void>;
|
|
59
|
+
readonly scenes: readonly VideoScene[];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function defineVideo(definition: VideoDefinition): VideoDefinition {
|
|
63
|
+
if (definition.scenes.length === 0) {
|
|
64
|
+
throw new Error("defineVideo(): a video needs at least one scene");
|
|
65
|
+
}
|
|
66
|
+
if (definition.viewport !== undefined) {
|
|
67
|
+
const { width, height } = definition.viewport;
|
|
68
|
+
// libx264 with yuv420p cannot encode odd dimensions.
|
|
69
|
+
if (width % 2 !== 0 || height % 2 !== 0) {
|
|
70
|
+
throw new Error(`defineVideo(): viewport ${width}x${height} must have even width and height`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const seen = new Set<string>();
|
|
74
|
+
for (const scene of definition.scenes) {
|
|
75
|
+
if (seen.has(scene.id)) throw new Error(`defineVideo(): duplicate scene id "${scene.id}"`);
|
|
76
|
+
seen.add(scene.id);
|
|
77
|
+
}
|
|
78
|
+
return definition;
|
|
79
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { Locator, Page } from "@playwright/test";
|
|
4
|
+
import { type PresentIdentity, withPresentedIdentities } from "../present-identities";
|
|
5
|
+
import { trackInFlightRequests, waitForSettledPage } from "../screenshots";
|
|
6
|
+
import { clearRipples, moveCursor, rippleAt, showFocus } from "./cursor-overlay";
|
|
7
|
+
import type { VideoTools } from "./define-video";
|
|
8
|
+
import { VIDEO_FPS } from "./mux";
|
|
9
|
+
|
|
10
|
+
export const FRAME_MS = 1000 / VIDEO_FPS;
|
|
11
|
+
const MOVE_STEPS = 15;
|
|
12
|
+
const RIPPLE_FRAMES = 6;
|
|
13
|
+
const TYPE_FRAME_MS = 60;
|
|
14
|
+
|
|
15
|
+
export interface TimelineFrame {
|
|
16
|
+
readonly path: string;
|
|
17
|
+
durationMs: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function easeInOut(t: number): number {
|
|
21
|
+
return t < 0.5 ? 2 * t * t : 1 - (-2 * t + 2) ** 2 / 2;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Virtual time: the page is never recorded live. Each screenshot is a frame
|
|
25
|
+
// with an assigned duration, so slow captures do not stretch the video.
|
|
26
|
+
export class FrameTimeline {
|
|
27
|
+
private readonly frames: TimelineFrame[] = [];
|
|
28
|
+
private cursor = { x: 0, y: 0 };
|
|
29
|
+
private stageClockMs = 0;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
private readonly page: Page,
|
|
33
|
+
private readonly frameDir: string,
|
|
34
|
+
private readonly identities: readonly PresentIdentity[] = [],
|
|
35
|
+
) {
|
|
36
|
+
mkdirSync(frameDir, { recursive: true });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
get allFrames(): readonly TimelineFrame[] {
|
|
40
|
+
return this.frames;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
get totalMs(): number {
|
|
44
|
+
return this.frames.reduce((sum, frame) => sum + frame.durationMs, 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async capture(durationMs: number, animations: "disabled" | "allow" = "disabled"): Promise<void> {
|
|
48
|
+
const path = join(
|
|
49
|
+
this.frameDir,
|
|
50
|
+
`frame-${String(this.frames.length + 1).padStart(5, "0")}.png`,
|
|
51
|
+
);
|
|
52
|
+
await withPresentedIdentities(this.page, this.identities, () =>
|
|
53
|
+
this.page.screenshot({ path, animations }),
|
|
54
|
+
);
|
|
55
|
+
this.frames.push({ path, durationMs });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
extendLast(ms: number): void {
|
|
59
|
+
const last = this.frames.at(-1);
|
|
60
|
+
if (last === undefined) throw new Error("FrameTimeline.extendLast(): no frame captured yet");
|
|
61
|
+
last.durationMs += ms;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async settle(): Promise<void> {
|
|
65
|
+
await waitForSettledPage(this.page, trackInFlightRequests(this.page));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private async centerOf(target: Locator): Promise<{ x: number; y: number }> {
|
|
69
|
+
await target.scrollIntoViewIfNeeded();
|
|
70
|
+
const box = await target.boundingBox();
|
|
71
|
+
if (box === null) throw new Error("Video tools: target element has no bounding box");
|
|
72
|
+
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private async glideTo(destination: { x: number; y: number }): Promise<void> {
|
|
76
|
+
const from = this.cursor;
|
|
77
|
+
for (let step = 1; step <= MOVE_STEPS; step++) {
|
|
78
|
+
const t = easeInOut(step / MOVE_STEPS);
|
|
79
|
+
const x = from.x + (destination.x - from.x) * t;
|
|
80
|
+
const y = from.y + (destination.y - from.y) * t;
|
|
81
|
+
await moveCursor(this.page, x, y);
|
|
82
|
+
await this.capture(FRAME_MS);
|
|
83
|
+
}
|
|
84
|
+
this.cursor = destination;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
readonly tools: VideoTools = {
|
|
88
|
+
moveTo: async (target) => {
|
|
89
|
+
await this.glideTo(await this.centerOf(target));
|
|
90
|
+
},
|
|
91
|
+
click: async (target) => {
|
|
92
|
+
await this.tools.moveTo(target);
|
|
93
|
+
await rippleAt(this.page, this.cursor.x, this.cursor.y);
|
|
94
|
+
for (let frame = 0; frame < RIPPLE_FRAMES; frame++) await this.capture(FRAME_MS);
|
|
95
|
+
await clearRipples(this.page);
|
|
96
|
+
await target.click();
|
|
97
|
+
await this.settle();
|
|
98
|
+
await this.capture(FRAME_MS);
|
|
99
|
+
},
|
|
100
|
+
type: async (target, text) => {
|
|
101
|
+
await this.tools.click(target);
|
|
102
|
+
for (const char of text) {
|
|
103
|
+
await target.pressSequentially(char);
|
|
104
|
+
await this.capture(TYPE_FRAME_MS);
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
hold: async (ms) => {
|
|
108
|
+
await this.capture(ms);
|
|
109
|
+
},
|
|
110
|
+
snapshot: async () => {
|
|
111
|
+
const png = await withPresentedIdentities(this.page, this.identities, () =>
|
|
112
|
+
this.page.screenshot({ animations: "disabled" }),
|
|
113
|
+
);
|
|
114
|
+
return `data:image/png;base64,${png.toString("base64")}`;
|
|
115
|
+
},
|
|
116
|
+
stage: async (html) => {
|
|
117
|
+
await this.page.setContent(html, { waitUntil: "load" });
|
|
118
|
+
this.stageClockMs = 0;
|
|
119
|
+
},
|
|
120
|
+
animate: async (ms) => {
|
|
121
|
+
const frames = Math.max(1, Math.round(ms / FRAME_MS));
|
|
122
|
+
for (let frame = 0; frame < frames; frame++) {
|
|
123
|
+
await this.page.evaluate((clockMs) => {
|
|
124
|
+
for (const animation of document.getAnimations()) {
|
|
125
|
+
animation.pause();
|
|
126
|
+
animation.currentTime = clockMs;
|
|
127
|
+
}
|
|
128
|
+
}, this.stageClockMs);
|
|
129
|
+
// "disabled" would fast-forward the paused animations to their end state.
|
|
130
|
+
await this.capture(FRAME_MS, "allow");
|
|
131
|
+
this.stageClockMs += FRAME_MS;
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
focus: async (target) => {
|
|
135
|
+
await this.settle();
|
|
136
|
+
await target.scrollIntoViewIfNeeded();
|
|
137
|
+
await showFocus(this.page, await target.boundingBox());
|
|
138
|
+
await this.capture(FRAME_MS);
|
|
139
|
+
},
|
|
140
|
+
};
|
|
141
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { mkdirSync, renameSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import type { VideoMusic } from "./define-video";
|
|
5
|
+
|
|
6
|
+
export const VIDEO_FPS = 30;
|
|
7
|
+
|
|
8
|
+
export interface MuxFrame {
|
|
9
|
+
readonly path: string;
|
|
10
|
+
readonly durationMs: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface VttCue {
|
|
14
|
+
readonly startMs: number;
|
|
15
|
+
readonly endMs: number;
|
|
16
|
+
readonly text: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface MuxSceneAudio {
|
|
20
|
+
readonly wavPath: string;
|
|
21
|
+
readonly startMs: number;
|
|
22
|
+
readonly durationMs: number;
|
|
23
|
+
readonly sceneMs: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function concatFileLine(path: string): string {
|
|
27
|
+
return `file '${path.replaceAll("'", "'\\''")}'`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function buildFrameConcatList(frames: readonly MuxFrame[]): string {
|
|
31
|
+
const last = frames.at(-1);
|
|
32
|
+
if (last === undefined) throw new Error("buildFrameConcatList(): no frames");
|
|
33
|
+
const lines = frames.flatMap((frame) => [
|
|
34
|
+
concatFileLine(frame.path),
|
|
35
|
+
`duration ${(frame.durationMs / 1000).toFixed(4)}`,
|
|
36
|
+
]);
|
|
37
|
+
// The concat demuxer ignores the duration of the final entry unless that file is listed again.
|
|
38
|
+
lines.push(concatFileLine(last.path));
|
|
39
|
+
return `${lines.join("\n")}\n`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function formatVttTimestamp(ms: number): string {
|
|
43
|
+
const total = Math.max(0, Math.round(ms));
|
|
44
|
+
const pad = (value: number, width: number): string => String(value).padStart(width, "0");
|
|
45
|
+
const hours = Math.floor(total / 3_600_000);
|
|
46
|
+
const minutes = Math.floor((total % 3_600_000) / 60_000);
|
|
47
|
+
const seconds = Math.floor((total % 60_000) / 1000);
|
|
48
|
+
return `${pad(hours, 2)}:${pad(minutes, 2)}:${pad(seconds, 2)}.${pad(total % 1000, 3)}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// A blank line ends a cue and "-->" starts a timing line, so neither may reach the cue text.
|
|
52
|
+
function vttCueText(text: string): string {
|
|
53
|
+
return text
|
|
54
|
+
.replaceAll("&", "&")
|
|
55
|
+
.replaceAll("<", "<")
|
|
56
|
+
.replaceAll("-->", "-->")
|
|
57
|
+
.replace(/\n\s*\n/g, "\n");
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function buildVtt(cues: readonly VttCue[]): string {
|
|
61
|
+
const blocks = cues.map(
|
|
62
|
+
(cue) =>
|
|
63
|
+
`${formatVttTimestamp(cue.startMs)} --> ${formatVttTimestamp(cue.endMs)}\n${vttCueText(cue.text)}`,
|
|
64
|
+
);
|
|
65
|
+
return `WEBVTT\n\n${blocks.join("\n\n")}\n`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Per scene: delay the narration to its start, then pad with silence to the
|
|
69
|
+
// scene's end so audio and picture stay aligned across scenes.
|
|
70
|
+
export function buildAudioFilter(
|
|
71
|
+
sceneAudio: readonly MuxSceneAudio[],
|
|
72
|
+
firstInputIndex: number,
|
|
73
|
+
outputLabel = "aout",
|
|
74
|
+
): string {
|
|
75
|
+
let previousEndMs = 0;
|
|
76
|
+
const chains = sceneAudio.map((scene, index) => {
|
|
77
|
+
const leadMs = Math.max(0, scene.startMs - previousEndMs);
|
|
78
|
+
const wholeSeconds = ((leadMs + scene.sceneMs) / 1000).toFixed(4);
|
|
79
|
+
previousEndMs = scene.startMs + scene.sceneMs;
|
|
80
|
+
const delay = leadMs > 0 ? `adelay=${leadMs}:all=1,` : "";
|
|
81
|
+
return `[${firstInputIndex + index}:a]${delay}apad=whole_dur=${wholeSeconds},atrim=0:${wholeSeconds}[a${index}]`;
|
|
82
|
+
});
|
|
83
|
+
const joined = sceneAudio.map((_, index) => `[a${index}]`).join("");
|
|
84
|
+
return `${chains.join(";")};${joined}concat=n=${sceneAudio.length}:v=0:a=1[${outputLabel}]`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export const DEFAULT_MUSIC_VOLUME = 0.35;
|
|
88
|
+
const MUSIC_FADE_OUT_SECONDS = 2;
|
|
89
|
+
|
|
90
|
+
// Music loops under the narration and is ducked by it via sidechain compression.
|
|
91
|
+
export function buildMusicMixFilter(
|
|
92
|
+
musicInputIndex: number,
|
|
93
|
+
volume: number,
|
|
94
|
+
videoMs: number,
|
|
95
|
+
): string {
|
|
96
|
+
const videoSeconds = videoMs / 1000;
|
|
97
|
+
const fadeStart = Math.max(0, videoSeconds - MUSIC_FADE_OUT_SECONDS).toFixed(4);
|
|
98
|
+
return [
|
|
99
|
+
"[voice]asplit=2[voiceMix][voiceSc]",
|
|
100
|
+
`[${musicInputIndex}:a]aloop=loop=-1:size=2e9,atrim=0:${videoSeconds.toFixed(4)},volume=${volume},afade=t=out:st=${fadeStart}:d=${MUSIC_FADE_OUT_SECONDS}[music]`,
|
|
101
|
+
"[music][voiceSc]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400[ducked]",
|
|
102
|
+
"[voiceMix][ducked]amix=inputs=2:duration=first:normalize=0[aout]",
|
|
103
|
+
].join(";");
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Video lives in filter_complex too: -vf cannot be mixed with -filter_complex.
|
|
107
|
+
export function buildVideoFilter(brandInputIndex: number | undefined): string {
|
|
108
|
+
if (brandInputIndex === undefined) return `[0:v]fps=${VIDEO_FPS}[vout]`;
|
|
109
|
+
return `[0:v]fps=${VIDEO_FPS}[base];[base][${brandInputIndex}:v]overlay=W-w-40:H-h-40[vout]`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface MuxVideoOptions {
|
|
113
|
+
readonly music?: VideoMusic;
|
|
114
|
+
readonly metadataComment?: string;
|
|
115
|
+
readonly brandPng?: string;
|
|
116
|
+
readonly frames: readonly MuxFrame[];
|
|
117
|
+
readonly sceneAudio: readonly MuxSceneAudio[];
|
|
118
|
+
readonly outMp4: string;
|
|
119
|
+
readonly workDir: string;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function muxVideo(options: MuxVideoOptions): void {
|
|
123
|
+
const { frames, outMp4, workDir } = options;
|
|
124
|
+
mkdirSync(workDir, { recursive: true });
|
|
125
|
+
mkdirSync(dirname(outMp4), { recursive: true });
|
|
126
|
+
const listPath = join(workDir, "frames.txt");
|
|
127
|
+
writeFileSync(listPath, buildFrameConcatList(frames));
|
|
128
|
+
// A player that has the old file open loses its audio when ffmpeg truncates it in place;
|
|
129
|
+
// the rename swaps in the new file while the player keeps reading the old one.
|
|
130
|
+
const partialMp4 = `${outMp4}.partial.mp4`;
|
|
131
|
+
const args = buildMuxArgs({ ...options, outMp4: partialMp4 }, listPath);
|
|
132
|
+
execFileSync("ffmpeg", args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
133
|
+
renameSync(partialMp4, outMp4);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function buildMuxArgs(options: MuxVideoOptions, listPath: string): string[] {
|
|
137
|
+
const { frames, sceneAudio, outMp4, brandPng, music, metadataComment } = options;
|
|
138
|
+
const audioInputs = sceneAudio.flatMap((scene) => ["-i", scene.wavPath]);
|
|
139
|
+
const musicInputIndex = music === undefined ? undefined : 1 + sceneAudio.length;
|
|
140
|
+
const brandInputIndex =
|
|
141
|
+
brandPng === undefined ? undefined : 1 + sceneAudio.length + (music === undefined ? 0 : 1);
|
|
142
|
+
const audioFilter =
|
|
143
|
+
music === undefined || musicInputIndex === undefined
|
|
144
|
+
? buildAudioFilter(sceneAudio, 1)
|
|
145
|
+
: `${buildAudioFilter(sceneAudio, 1, "voice")};${buildMusicMixFilter(
|
|
146
|
+
musicInputIndex,
|
|
147
|
+
music.volume ?? DEFAULT_MUSIC_VOLUME,
|
|
148
|
+
frames.reduce((total, frame) => total + frame.durationMs, 0),
|
|
149
|
+
)}`;
|
|
150
|
+
return [
|
|
151
|
+
"-y",
|
|
152
|
+
"-f",
|
|
153
|
+
"concat",
|
|
154
|
+
"-safe",
|
|
155
|
+
"0",
|
|
156
|
+
"-i",
|
|
157
|
+
listPath,
|
|
158
|
+
...audioInputs,
|
|
159
|
+
...(music === undefined ? [] : ["-i", music.file]),
|
|
160
|
+
...(brandPng === undefined ? [] : ["-i", brandPng]),
|
|
161
|
+
"-filter_complex",
|
|
162
|
+
`${buildVideoFilter(brandInputIndex)};${audioFilter}`,
|
|
163
|
+
"-map",
|
|
164
|
+
"[vout]",
|
|
165
|
+
"-map",
|
|
166
|
+
"[aout]",
|
|
167
|
+
"-c:v",
|
|
168
|
+
"libx264",
|
|
169
|
+
// CRF 28 halves the size of animated stages at no visible loss; faststart puts the
|
|
170
|
+
// index up front so a browser starts playing before the download ends.
|
|
171
|
+
"-preset",
|
|
172
|
+
"slow",
|
|
173
|
+
"-crf",
|
|
174
|
+
"28",
|
|
175
|
+
"-pix_fmt",
|
|
176
|
+
"yuv420p",
|
|
177
|
+
"-c:a",
|
|
178
|
+
"aac",
|
|
179
|
+
"-shortest",
|
|
180
|
+
"-movflags",
|
|
181
|
+
"+faststart",
|
|
182
|
+
...(metadataComment === undefined ? [] : ["-metadata", `comment=${metadataComment}`]),
|
|
183
|
+
outMp4,
|
|
184
|
+
];
|
|
185
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { extname, join } from "node:path";
|
|
3
|
+
import type { Page } from "@playwright/test";
|
|
4
|
+
import { DESKTOP_VIEWPORT } from "../constants";
|
|
5
|
+
import type { PresentIdentity } from "../present-identities";
|
|
6
|
+
import { assertPresentableMappings } from "../present-identities";
|
|
7
|
+
import { resolveLocaleTag, trackInFlightRequests } from "../screenshots";
|
|
8
|
+
import { test } from "../seeded-tenant-fixture";
|
|
9
|
+
import { addVideoOverlay, showFocus } from "./cursor-overlay";
|
|
10
|
+
import type { VideoDefinition, VideoSceneContext } from "./define-video";
|
|
11
|
+
import { FrameTimeline } from "./frame-timeline";
|
|
12
|
+
import { buildVtt, type MuxSceneAudio, muxVideo, type VttCue } from "./mux";
|
|
13
|
+
import { synthesizeSpeech } from "./tts";
|
|
14
|
+
import { loadVideoTexts } from "./video-texts";
|
|
15
|
+
|
|
16
|
+
const INTRO_MS = 500;
|
|
17
|
+
const SCENE_TAIL_MS = 400;
|
|
18
|
+
|
|
19
|
+
export interface RunVideosOptions {
|
|
20
|
+
readonly videos: Readonly<Record<string, VideoDefinition>>;
|
|
21
|
+
readonly videosDir: string;
|
|
22
|
+
readonly locales: readonly string[];
|
|
23
|
+
/** Absolute path to an SVG or PNG, overlaid bottom right on every video. */
|
|
24
|
+
readonly brandLogo?: string;
|
|
25
|
+
// Locale to disclosure text ("AI voice"); shown next to the logo and written to the MP4 comment.
|
|
26
|
+
readonly aiNotice?: Readonly<Record<string, string>>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function escapeHtml(text: string): string {
|
|
30
|
+
return text
|
|
31
|
+
.replaceAll("&", "&")
|
|
32
|
+
.replaceAll("<", "<")
|
|
33
|
+
.replaceAll(">", ">")
|
|
34
|
+
.replaceAll('"', """);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const BRAND_MIME: Readonly<Record<string, string>> = {
|
|
38
|
+
".svg": "image/svg+xml",
|
|
39
|
+
".png": "image/png",
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
async function renderBrandPng(
|
|
43
|
+
page: Page,
|
|
44
|
+
logoPath: string,
|
|
45
|
+
outPng: string,
|
|
46
|
+
notice: string | undefined,
|
|
47
|
+
): Promise<void> {
|
|
48
|
+
const mime = BRAND_MIME[extname(logoPath).toLowerCase()];
|
|
49
|
+
if (mime === undefined) throw new Error(`brandLogo "${logoPath}": use a .svg or .png file`);
|
|
50
|
+
const dataUrl = `data:${mime};base64,${readFileSync(logoPath).toString("base64")}`;
|
|
51
|
+
const noticeHtml =
|
|
52
|
+
notice === undefined
|
|
53
|
+
? ""
|
|
54
|
+
: `<span style="width:1px;height:28px;background:rgba(31,59,92,0.25)"></span><span style="font:18px system-ui,-apple-system,'Segoe UI',sans-serif;color:rgba(31,59,92,0.75);white-space:nowrap">${escapeHtml(notice)}</span>`;
|
|
55
|
+
await page.setContent(
|
|
56
|
+
`<body style="margin:0;background:transparent"><div id="brand" style="display:inline-flex;align-items:center;gap:16px;margin:20px;padding:12px 20px;border-radius:16px;background:rgba(246,244,239,0.9);box-shadow:0 2px 8px rgba(0,0,0,0.18)"><img src="${escapeHtml(dataUrl)}" style="display:block;height:40px">${noticeHtml}</div></body>`,
|
|
57
|
+
);
|
|
58
|
+
await page.locator("#brand").screenshot({ path: outPng, omitBackground: true });
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function csvFilter(raw: string | undefined): ReadonlySet<string> | undefined {
|
|
62
|
+
if (raw === undefined || raw.trim() === "") return undefined;
|
|
63
|
+
return new Set(
|
|
64
|
+
raw
|
|
65
|
+
.split(",")
|
|
66
|
+
.map((entry) => entry.trim())
|
|
67
|
+
.filter((entry) => entry !== ""),
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Registrar like runMatrix: call at the spec's module top, do NOT await.
|
|
72
|
+
export function runVideos(options: RunVideosOptions): void {
|
|
73
|
+
const { videos, videosDir, locales, brandLogo, aiNotice } = options;
|
|
74
|
+
const onlyIds = csvFilter(process.env["VIDEO_ONLY"]);
|
|
75
|
+
const onlyLocales = csvFilter(process.env["VIDEO_LOCALES"]);
|
|
76
|
+
const ttsCacheDir = join(videosDir, ".cache", "tts");
|
|
77
|
+
|
|
78
|
+
if (aiNotice !== undefined) {
|
|
79
|
+
for (const locale of locales) {
|
|
80
|
+
if (aiNotice[locale] === undefined) {
|
|
81
|
+
throw new Error(`runVideos(): aiNotice has no text for locale "${locale}"`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (const [id, video] of Object.entries(videos)) {
|
|
87
|
+
if (onlyIds !== undefined && !onlyIds.has(id)) continue;
|
|
88
|
+
for (const locale of locales) {
|
|
89
|
+
if (onlyLocales !== undefined && !onlyLocales.has(locale)) continue;
|
|
90
|
+
test.describe(locale, () => {
|
|
91
|
+
test.use({ locale: resolveLocaleTag(locale, undefined) });
|
|
92
|
+
test(`${id} [${locale}]`, async ({ page, seedTenant }, testInfo) => {
|
|
93
|
+
const videoDir = join(videosDir, id);
|
|
94
|
+
const texts = loadVideoTexts(videoDir, locale);
|
|
95
|
+
await page.setViewportSize(video.viewport ?? DESKTOP_VIEWPORT);
|
|
96
|
+
await page.emulateMedia({ reducedMotion: "reduce" });
|
|
97
|
+
await page.addInitScript((lng) => {
|
|
98
|
+
localStorage.setItem("kumiko:locale", lng);
|
|
99
|
+
}, locale);
|
|
100
|
+
await addVideoOverlay(page);
|
|
101
|
+
// Count requests fired by `setup` in the first settle().
|
|
102
|
+
trackInFlightRequests(page);
|
|
103
|
+
|
|
104
|
+
const identities: PresentIdentity[] = [];
|
|
105
|
+
const timeline = new FrameTimeline(page, testInfo.outputPath("frames"), identities);
|
|
106
|
+
const ctx: VideoSceneContext = {
|
|
107
|
+
seedTenant,
|
|
108
|
+
locale,
|
|
109
|
+
tools: timeline.tools,
|
|
110
|
+
state: new Map(),
|
|
111
|
+
speechMs: 0,
|
|
112
|
+
presentIdentities: (mappings) => {
|
|
113
|
+
assertPresentableMappings(mappings);
|
|
114
|
+
identities.push(...mappings);
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
await video.setup?.(page, ctx);
|
|
118
|
+
|
|
119
|
+
await timeline.settle();
|
|
120
|
+
await timeline.capture(INTRO_MS);
|
|
121
|
+
|
|
122
|
+
const sceneAudio: MuxSceneAudio[] = [];
|
|
123
|
+
const cues: VttCue[] = [];
|
|
124
|
+
for (const scene of video.scenes) {
|
|
125
|
+
const text = texts[scene.id];
|
|
126
|
+
if (text === undefined)
|
|
127
|
+
throw new Error(`Video "${id}" [${locale}]: scene "${scene.id}" has no text`);
|
|
128
|
+
const speech = await synthesizeSpeech({
|
|
129
|
+
text,
|
|
130
|
+
voice: video.voice,
|
|
131
|
+
...(scene.instructions !== undefined && { sceneInstructions: scene.instructions }),
|
|
132
|
+
cacheDir: ttsCacheDir,
|
|
133
|
+
apiKey: process.env["OPENROUTER_API_KEY"],
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
// Narration starts at the scene start, so the intro frame stays silent.
|
|
137
|
+
const startMs = timeline.totalMs;
|
|
138
|
+
await scene.do(page, { ...ctx, speechMs: speech.durationMs });
|
|
139
|
+
const minimumMs = speech.durationMs + SCENE_TAIL_MS;
|
|
140
|
+
const elapsedMs = timeline.totalMs - startMs;
|
|
141
|
+
if (elapsedMs < minimumMs) timeline.extendLast(minimumMs - elapsedMs);
|
|
142
|
+
await showFocus(page, null);
|
|
143
|
+
|
|
144
|
+
const sceneMs = timeline.totalMs - startMs;
|
|
145
|
+
sceneAudio.push({
|
|
146
|
+
wavPath: speech.wavPath,
|
|
147
|
+
startMs,
|
|
148
|
+
durationMs: speech.durationMs,
|
|
149
|
+
sceneMs,
|
|
150
|
+
});
|
|
151
|
+
cues.push({ startMs, endMs: startMs + sceneMs, text });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const outDir = join(videoDir, "out");
|
|
155
|
+
mkdirSync(outDir, { recursive: true });
|
|
156
|
+
const outMp4 = join(outDir, `${locale}.mp4`);
|
|
157
|
+
const notice = aiNotice?.[locale];
|
|
158
|
+
const muxDir = testInfo.outputPath("mux");
|
|
159
|
+
mkdirSync(muxDir, { recursive: true });
|
|
160
|
+
let brandPng: string | undefined;
|
|
161
|
+
if (brandLogo !== undefined) {
|
|
162
|
+
brandPng = join(muxDir, "brand.png");
|
|
163
|
+
await renderBrandPng(page, brandLogo, brandPng, notice);
|
|
164
|
+
}
|
|
165
|
+
muxVideo({
|
|
166
|
+
...(brandPng !== undefined && { brandPng }),
|
|
167
|
+
...(notice !== undefined && { metadataComment: notice }),
|
|
168
|
+
...(video.music !== undefined && { music: video.music }),
|
|
169
|
+
frames: timeline.allFrames,
|
|
170
|
+
sceneAudio,
|
|
171
|
+
outMp4,
|
|
172
|
+
workDir: muxDir,
|
|
173
|
+
});
|
|
174
|
+
writeFileSync(join(outDir, `${locale}.vtt`), buildVtt(cues));
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { DEFAULT_TTS_MODEL, type VideoVoice } from "./define-video";
|
|
5
|
+
|
|
6
|
+
const SAMPLE_RATE = 24_000;
|
|
7
|
+
const BYTES_PER_SAMPLE = 2;
|
|
8
|
+
const WAV_HEADER_BYTES = 44;
|
|
9
|
+
const SPEECH_URL = "https://openrouter.ai/api/v1/audio/speech";
|
|
10
|
+
const ERROR_BODY_CHARS = 300;
|
|
11
|
+
|
|
12
|
+
export function pcmDurationMs(byteLength: number, sampleRate = SAMPLE_RATE): number {
|
|
13
|
+
return Math.round((byteLength / BYTES_PER_SAMPLE / sampleRate) * 1000);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function pcmToWav(pcm: Uint8Array, sampleRate = SAMPLE_RATE): Uint8Array {
|
|
17
|
+
const wav = new Uint8Array(WAV_HEADER_BYTES + pcm.byteLength);
|
|
18
|
+
const view = new DataView(wav.buffer);
|
|
19
|
+
const writeTag = (offset: number, tag: string): void => {
|
|
20
|
+
for (let i = 0; i < tag.length; i++) view.setUint8(offset + i, tag.charCodeAt(i));
|
|
21
|
+
};
|
|
22
|
+
writeTag(0, "RIFF");
|
|
23
|
+
view.setUint32(4, 36 + pcm.byteLength, true);
|
|
24
|
+
writeTag(8, "WAVE");
|
|
25
|
+
writeTag(12, "fmt ");
|
|
26
|
+
view.setUint32(16, 16, true);
|
|
27
|
+
view.setUint16(20, 1, true);
|
|
28
|
+
view.setUint16(22, 1, true);
|
|
29
|
+
view.setUint32(24, sampleRate, true);
|
|
30
|
+
view.setUint32(28, sampleRate * BYTES_PER_SAMPLE, true);
|
|
31
|
+
view.setUint16(32, BYTES_PER_SAMPLE, true);
|
|
32
|
+
view.setUint16(34, 16, true);
|
|
33
|
+
writeTag(36, "data");
|
|
34
|
+
view.setUint32(40, pcm.byteLength, true);
|
|
35
|
+
wav.set(pcm, WAV_HEADER_BYTES);
|
|
36
|
+
return wav;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface SynthesizeSpeechOptions {
|
|
40
|
+
readonly text: string;
|
|
41
|
+
readonly voice: VideoVoice;
|
|
42
|
+
readonly sceneInstructions?: string;
|
|
43
|
+
readonly cacheDir: string;
|
|
44
|
+
readonly apiKey: string | undefined;
|
|
45
|
+
readonly fetchImpl?: (url: string, init: RequestInit) => Promise<Response>;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface SynthesizedSpeech {
|
|
49
|
+
readonly wavPath: string;
|
|
50
|
+
readonly durationMs: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function synthesizeSpeech(
|
|
54
|
+
options: SynthesizeSpeechOptions,
|
|
55
|
+
): Promise<SynthesizedSpeech> {
|
|
56
|
+
const { text, voice, cacheDir, apiKey, fetchImpl = fetch } = options;
|
|
57
|
+
const model = voice.model ?? DEFAULT_TTS_MODEL;
|
|
58
|
+
const instructions = options.sceneInstructions ?? voice.instructions;
|
|
59
|
+
const hash = createHash("sha256")
|
|
60
|
+
.update(JSON.stringify({ model, voice: voice.voice, instructions: instructions ?? null, text }))
|
|
61
|
+
.digest("hex");
|
|
62
|
+
const wavPath = join(cacheDir, `${hash}.wav`);
|
|
63
|
+
if (existsSync(wavPath)) {
|
|
64
|
+
return { wavPath, durationMs: pcmDurationMs(statSync(wavPath).size - WAV_HEADER_BYTES) };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (apiKey === undefined || apiKey === "") throw new Error("OPENROUTER_API_KEY is missing");
|
|
68
|
+
// Gemini TTS only serves raw PCM (mp3 answers 400); instructions stay out of
|
|
69
|
+
// `input` because the model would read them aloud.
|
|
70
|
+
const response = await fetchImpl(SPEECH_URL, {
|
|
71
|
+
method: "POST",
|
|
72
|
+
headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
|
|
73
|
+
body: JSON.stringify({
|
|
74
|
+
model,
|
|
75
|
+
input: text,
|
|
76
|
+
voice: voice.voice,
|
|
77
|
+
response_format: "pcm",
|
|
78
|
+
...(instructions !== undefined && { instructions }),
|
|
79
|
+
}),
|
|
80
|
+
});
|
|
81
|
+
if (!response.ok) {
|
|
82
|
+
const body = (await response.text()).slice(0, ERROR_BODY_CHARS);
|
|
83
|
+
throw new Error(`TTS request failed with status ${response.status}: ${body}`);
|
|
84
|
+
}
|
|
85
|
+
const pcm = new Uint8Array(await response.arrayBuffer());
|
|
86
|
+
if (pcm.byteLength === 0 || pcm.byteLength % BYTES_PER_SAMPLE !== 0) {
|
|
87
|
+
throw new Error(`TTS answer is not valid s16le PCM (${pcm.byteLength} bytes)`);
|
|
88
|
+
}
|
|
89
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
90
|
+
// Atomic write: a killed run must not leave a truncated wav that later counts as a cache hit.
|
|
91
|
+
const tempPath = `${wavPath}.${process.pid}.tmp`;
|
|
92
|
+
writeFileSync(tempPath, pcmToWav(pcm));
|
|
93
|
+
renameSync(tempPath, wavPath);
|
|
94
|
+
return { wavPath, durationMs: pcmDurationMs(pcm.byteLength) };
|
|
95
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import * as z from "zod";
|
|
4
|
+
import type { VideoDefinition } from "./define-video";
|
|
5
|
+
|
|
6
|
+
const textsSchema = z.record(z.string(), z.string().min(1));
|
|
7
|
+
|
|
8
|
+
export function videoTextsPath(videoDir: string, locale: string): string {
|
|
9
|
+
return join(videoDir, `text.${locale}.yaml`);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function loadVideoTexts(videoDir: string, locale: string): Readonly<Record<string, string>> {
|
|
13
|
+
const path = videoTextsPath(videoDir, locale);
|
|
14
|
+
if (!existsSync(path)) throw new Error(`Video texts file missing: ${path}`);
|
|
15
|
+
// Bun API is safe here: Playwright runs as `bunx --bun playwright` (scaffold standard).
|
|
16
|
+
const parsed = textsSchema.safeParse(Bun.YAML.parse(readFileSync(path, "utf8")));
|
|
17
|
+
if (!parsed.success) {
|
|
18
|
+
throw new Error(
|
|
19
|
+
`Video texts invalid in ${path} (expected scene id -> non-empty text): ${parsed.error.message}`,
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
return parsed.data;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function findVideoTextGaps(
|
|
26
|
+
video: Pick<VideoDefinition, "scenes">,
|
|
27
|
+
videoDir: string,
|
|
28
|
+
locales: readonly string[],
|
|
29
|
+
): string[] {
|
|
30
|
+
const gaps: string[] = [];
|
|
31
|
+
const sceneIds = new Set(video.scenes.map((scene) => scene.id));
|
|
32
|
+
for (const locale of locales) {
|
|
33
|
+
const path = videoTextsPath(videoDir, locale);
|
|
34
|
+
if (!existsSync(path)) {
|
|
35
|
+
gaps.push(`${locale}: texts file missing (${path})`);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const texts = loadVideoTexts(videoDir, locale);
|
|
39
|
+
for (const id of sceneIds) {
|
|
40
|
+
if (texts[id] === undefined) gaps.push(`${locale}: scene "${id}" has no text`);
|
|
41
|
+
}
|
|
42
|
+
for (const id of Object.keys(texts)) {
|
|
43
|
+
if (!sceneIds.has(id)) gaps.push(`${locale}: text "${id}" has no scene`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return gaps;
|
|
47
|
+
}
|