@amaster.ai/pi-video-gen 0.1.29 → 0.1.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/remotion.d.ts +5 -0
- package/dist/remotion.d.ts.map +1 -1
- package/dist/remotion.js +94 -6
- package/dist/remotion.js.map +1 -1
- package/dist/render.d.ts +1 -0
- package/dist/render.d.ts.map +1 -1
- package/dist/render.js +18 -7
- package/dist/render.js.map +1 -1
- package/package.json +6 -6
- package/skills/video-gen/SKILL.md +30 -207
- package/skills/video-gen/evals.json +53 -24
- package/skills/video-gen/examples/continuous-demo/README.md +37 -0
- package/skills/video-gen/examples/continuous-demo/package.json +19 -0
- package/skills/video-gen/examples/continuous-demo/prepare-audio.mjs +66 -0
- package/skills/video-gen/examples/continuous-demo/src/index.tsx +151 -0
- package/skills/video-gen/examples/continuous-demo/timing.mjs +19 -0
- package/skills/video-gen/references/film-direction.md +33 -4
- package/skills/video-gen/references/motion-authoring.md +31 -0
- package/skills/video-gen/references/remotion-handoff.md +12 -0
- package/skills/video-gen/references/seedance-assets.md +16 -0
- package/skills/video-gen/references/seedance-public-material-library.md +2 -2
- package/skills/video-gen/references/shot-planning.md +68 -0
- package/skills/video-gen/references/timeline-compose.md +40 -0
- package/skills/video-gen/references/video-render-workflow.md +40 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import { AbsoluteFill, Audio, Composition, Easing, interpolate, registerRoot, staticFile, useCurrentFrame, useVideoConfig } from 'remotion';
|
|
3
|
+
import { durationSec, events, poses } from '../timing.mjs';
|
|
4
|
+
|
|
5
|
+
const blue = '#275de9';
|
|
6
|
+
const ink = '#12203c';
|
|
7
|
+
const ease = Easing.bezier(0.65, 0, 0.2, 1);
|
|
8
|
+
const clamp = { extrapolateLeft: 'clamp', extrapolateRight: 'clamp' } as const;
|
|
9
|
+
const move = (t: number, from: number, to: number) => interpolate(t, [from, to], [0, 1], { ...clamp, easing: ease });
|
|
10
|
+
const visible = (t: number, start: number, end: number) => move(t, start, start + 0.45) * (1 - move(t, end - 0.4, end));
|
|
11
|
+
|
|
12
|
+
// Procedural footage: original geometry, deterministic at every frame, no remote assets.
|
|
13
|
+
const Artwork = ({ t, variant = 0, labelOpacity = 1 }: { t: number; variant?: number; labelOpacity?: number }) => {
|
|
14
|
+
const palettes = [
|
|
15
|
+
['#061d4b', '#2264ef', '#83d9f7'],
|
|
16
|
+
['#3c1646', '#ed7a66', '#fbd6ac'],
|
|
17
|
+
['#0e4348', '#31b9a5', '#a5ead7'],
|
|
18
|
+
];
|
|
19
|
+
const [dark, middle, light] = palettes[variant % 3]!;
|
|
20
|
+
return <AbsoluteFill style={{ overflow: 'hidden', background: dark }}>
|
|
21
|
+
<svg viewBox="0 0 1000 600" preserveAspectRatio="xMidYMid slice" style={{ width: '100%', height: '100%' }}>
|
|
22
|
+
<defs>
|
|
23
|
+
<radialGradient id={`orb-${variant}`} cx="36%" cy="26%" r="76%">
|
|
24
|
+
<stop offset="0" stopColor={light} /><stop offset="0.42" stopColor={middle} /><stop offset="1" stopColor={dark} />
|
|
25
|
+
</radialGradient>
|
|
26
|
+
<linearGradient id={`ground-${variant}`} x1="0" y1="0" x2="0" y2="1">
|
|
27
|
+
<stop stopColor={middle} stopOpacity="0" /><stop offset="1" stopColor={middle} stopOpacity="0.5" />
|
|
28
|
+
</linearGradient>
|
|
29
|
+
</defs>
|
|
30
|
+
<path d="M0 400 Q500 260 1000 410 L1000 600 L0 600Z" fill={`url(#ground-${variant})`} />
|
|
31
|
+
{Array.from({ length: 17 }, (_, i) => <path key={i} d={`M-100 ${420 + i * 18} Q500 ${270 + i * 20} 1100 ${430 + i * 18}`} fill="none" stroke={light} opacity={0.05 + i * 0.003} strokeWidth="1" />)}
|
|
32
|
+
{variant === 0 && <g transform={`translate(${620 + Math.sin(t * 0.65 + variant) * 28} ${270 + Math.cos(t * 0.6) * 12}) rotate(${-22 + t * 4 + variant * 18})`}>
|
|
33
|
+
<circle r="167" fill={`url(#orb-${variant})`} />
|
|
34
|
+
{Array.from({ length: 19 }, (_, i) => <ellipse key={i} rx={30 + i * 7.5} ry="168" transform={`rotate(${i * 9.5 + Math.sin(t * 0.4) * 14})`} fill="none" stroke={light} strokeWidth={i % 4 === 0 ? 1.6 : 0.8} opacity="0.23" />)}
|
|
35
|
+
<ellipse rx="236" ry="53" fill="none" stroke={light} strokeWidth="9" transform={`rotate(${18 + Math.sin(t * 0.5) * 12})`} opacity="0.9" />
|
|
36
|
+
<ellipse rx="252" ry="68" fill="none" stroke={light} strokeWidth="1" opacity="0.28" />
|
|
37
|
+
<circle cx="-232" cy="-20" r="11" fill={light} />
|
|
38
|
+
</g>}
|
|
39
|
+
{variant === 1 && <g transform="translate(500 280) rotate(-18)">
|
|
40
|
+
{Array.from({ length: 44 }, (_, i) => {
|
|
41
|
+
const phase = i / 43 * Math.PI * 2 + t * 0.35;
|
|
42
|
+
const x = Math.sin(phase) * 160;
|
|
43
|
+
const y = Math.cos(phase) * 150;
|
|
44
|
+
return <path key={i} d={`M-390 ${y * 0.4} C-240 ${y - 220} ${x - 150} ${-y} ${x} ${y} S300 ${y + 210} 390 ${-y * 0.4}`} fill="none" stroke={light} strokeWidth={i % 5 === 0 ? 4 : 1.3} opacity={0.3 + Math.sin(i / 44 * Math.PI) * 0.55} />;
|
|
45
|
+
})}
|
|
46
|
+
</g>}
|
|
47
|
+
{variant === 2 && <g transform="translate(500 110)">
|
|
48
|
+
{Array.from({ length: 25 }, (_, i) => {
|
|
49
|
+
const col = i % 5, row = Math.floor(i / 5);
|
|
50
|
+
const x = (col - row) * 58, y = (col + row) * 27;
|
|
51
|
+
const h = 35 + (1 + Math.sin(col * 0.8 + row * 0.7 + t * 0.7)) * 75;
|
|
52
|
+
return <g key={i} transform={`translate(${x} ${y})`}>
|
|
53
|
+
<path d={`M0 ${-h} L54 ${27-h} L0 ${54-h} L-54 ${27-h}Z`} fill={light} />
|
|
54
|
+
<path d={`M-54 ${27-h}L0 ${54-h}L0 54L-54 27Z`} fill={middle} />
|
|
55
|
+
<path d={`M0 ${54-h}L54 ${27-h}L54 27L0 54Z`} fill="#258a89" />
|
|
56
|
+
</g>;
|
|
57
|
+
})}
|
|
58
|
+
</g>}
|
|
59
|
+
{Array.from({ length: 18 }, (_, i) => <circle key={i} cx={(i * 151 + t * (3 + i % 4)) % 1050} cy={60 + (i * 79) % 440} r={i % 3 === 0 ? 2.4 : 1.2} fill={light} opacity="0.4" />)}
|
|
60
|
+
</svg>
|
|
61
|
+
<div style={{ position: 'absolute', left: '6%', bottom: '8%', color: '#fff', fontSize: 23, letterSpacing: 4, opacity: 0.8 * labelOpacity }}>PI / MOTION MATERIAL {String(variant + 1).padStart(2, '0')}</div>
|
|
62
|
+
</AbsoluteFill>;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const Heading = ({ t, start, end, label, children }: { t: number; start: number; end: number; label: string; children: React.ReactNode }) => {
|
|
66
|
+
const opacity = visible(t, start, end);
|
|
67
|
+
return <div style={{ position: 'absolute', left: 112, top: 266, opacity, transform: `translateY(${(1 - move(t, start, start + 0.7)) * 42}px)`, width: 610 }}>
|
|
68
|
+
<div style={{ fontFamily: 'monospace', fontSize: 24, letterSpacing: 3, color: blue, marginBottom: 34 }}>{label}</div>
|
|
69
|
+
<div style={{ fontSize: 82, fontWeight: 650, letterSpacing: -4, lineHeight: 1.23, whiteSpace: 'pre-line' }}>{children}</div>
|
|
70
|
+
</div>;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const Timeline = ({ t }: { t: number }) => {
|
|
74
|
+
const show = visible(t, 10.3, 14.2);
|
|
75
|
+
const reveal = move(t, 10.3, events.edit);
|
|
76
|
+
const play = move(t, events.edit, 17.4);
|
|
77
|
+
return <div style={{ position: 'absolute', left: 112, right: 112, top: 770, height: 220, background: '#fff', border: '1px solid #dce4f1', borderRadius: 20, opacity: show, transform: `translateY(${(1 - reveal) * 110}px)`, overflow: 'hidden', boxShadow: '0 20px 70px #13316312' }}>
|
|
78
|
+
<div style={{ position: 'absolute', left: 132, right: 25, top: 20, display: 'flex', justifyContent: 'space-between', font: '19px monospace', color: '#7e8aa0' }}>{['00:00', '00:04', '00:08', '00:12', '00:16', '00:20'].map((s) => <span key={s}>{s}</span>)}</div>
|
|
79
|
+
{['画面', '声音', '字幕'].map((label, row) => <div key={label} style={{ position: 'absolute', left: 22, top: 61 + row * 48, fontSize: 22, color: '#62708a' }}>{label}</div>)}
|
|
80
|
+
<div style={{ position: 'absolute', left: 128, top: 52, right: 24, height: 46, display: 'flex', gap: 7 }}>
|
|
81
|
+
{['想法', '素材展开', '进入镜头', '多轨编排', '审阅交付'].map((label, i) => <div key={label} style={{ flex: i === 1 ? 1.5 : 1, background: i === 2 ? blue : '#e6edfc', borderRadius: 7, padding: '10px 18px', fontSize: 21, color: i === 2 ? '#fff' : '#3b5b96' }}>{label}</div>)}
|
|
82
|
+
</div>
|
|
83
|
+
<svg viewBox="0 0 1500 42" style={{ position: 'absolute', left: 128, right: 24, top: 108, width: 1535, height: 40 }}>
|
|
84
|
+
{Array.from({ length: 140 }, (_, i) => { const a = 4 + Math.abs(Math.sin(i * 0.83) * Math.cos(i * 0.17)) * 32; return <rect key={i} x={i * 10.6} y={(42 - a) / 2} width="5" height={a} rx="2" fill="#4ba5b0" />; })}
|
|
85
|
+
</svg>
|
|
86
|
+
<div style={{ position: 'absolute', left: 128, right: 24, top: 160, height: 30, borderRadius: 6, background: '#f2d8bd', opacity: 0.8 }} />
|
|
87
|
+
<div style={{ position: 'absolute', top: 42, bottom: 12, left: 128 + play * 1510, width: 3, background: '#e78553', boxShadow: '0 0 16px #e7855355' }}><div style={{ width: 12, height: 12, background: '#e78553', marginLeft: -4.5, borderRadius: 3 }} /></div>
|
|
88
|
+
</div>;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const Review = ({ t }: { t: number }) => {
|
|
92
|
+
const opacity = visible(t, 18, 22);
|
|
93
|
+
return <div style={{ opacity }}>
|
|
94
|
+
<div style={{ position: 'absolute', left: 116, top: 592, color: '#62708a', fontSize: 27, lineHeight: 2.1 }}>
|
|
95
|
+
{['成片哈希与产物校验', '关键转场前、中、后抽帧', '源码和素材一起交付'].map((text, i) => <div key={text} style={{ opacity: move(t, 18.6 + i * 0.3, 19 + i * 0.3), transform: `translateX(${(1 - move(t, 18.6 + i * 0.3, 19 + i * 0.3)) * 22}px)` }}><span style={{ color: blue, marginRight: 20 }}>✓</span>{text}</div>)}
|
|
96
|
+
</div>
|
|
97
|
+
<div style={{ position: 'absolute', left: 794, right: 180, top: 730, display: 'flex', gap: 16 }}>
|
|
98
|
+
{['转场前', '转场中', '转场后'].map((label, i) => <div key={label} style={{ flex: 1, opacity: move(t, 18.4 + i * 0.14, 18.9 + i * 0.14) }}>
|
|
99
|
+
<div style={{ position: 'relative', height: 146, borderRadius: 12, overflow: 'hidden', border: i === 1 ? `3px solid ${blue}` : '3px solid #fff' }}><Artwork t={6.7 + i * 0.3} variant={0} labelOpacity={0} /></div>
|
|
100
|
+
<div style={{ paddingTop: 14, textAlign: 'center', fontSize: 22, color: '#62708a' }}>{label}</div>
|
|
101
|
+
</div>)}
|
|
102
|
+
</div>
|
|
103
|
+
</div>;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
const Demo = () => {
|
|
107
|
+
const frame = useCurrentFrame();
|
|
108
|
+
const { fps } = useVideoConfig();
|
|
109
|
+
const t = frame / fps;
|
|
110
|
+
const pose = Object.fromEntries(['x', 'y', 'w', 'h', 'rx', 'ry', 'rz'].map((key) => [key, interpolate(t, poses.map((p) => p.at), poses.map((p) => p[key]), { ...clamp, easing: ease })]));
|
|
111
|
+
const fan = visible(t, 2.9, 6.9);
|
|
112
|
+
const full = move(t, 6.2, events.enter) * (1 - move(t, 9.5, events.edit));
|
|
113
|
+
const layers = visible(t, 13.8, 18);
|
|
114
|
+
const close = move(t, 21.2, events.resolve);
|
|
115
|
+
const radius = (1 - full) * (26 + close * 15);
|
|
116
|
+
const light = '#f1f4fa';
|
|
117
|
+
return <AbsoluteFill style={{ background: light, color: ink, fontFamily: '"PingFang SC", "Microsoft YaHei", sans-serif', overflow: 'hidden' }}>
|
|
118
|
+
<div style={{ position: 'absolute', inset: 0, backgroundImage: 'radial-gradient(#c8d2e4 1px, transparent 1px)', backgroundSize: '32px 32px', opacity: 0.27 }} />
|
|
119
|
+
<div style={{ position: 'absolute', left: 112, right: 112, top: 62, display: 'flex', justifyContent: 'space-between', fontFamily: 'monospace', fontSize: 24, letterSpacing: 2, color: '#6c7c96', opacity: 1 - full }}><span>PI / VIDEO GEN</span><span>LOCAL MOTION FILM · 026</span></div>
|
|
120
|
+
<Heading t={t} start={-0.4} end={3.2} label="IDEA → MOTION">{'把想法,\n变成镜头。'}</Heading>
|
|
121
|
+
<div style={{ position: 'absolute', left: 116, top: 560, opacity: visible(t, 0.3, 3.1), color: '#62708a', fontSize: 31, lineHeight: 1.8 }}>素材、生成、剪辑、审阅。<br />在同一个创作流程里完成。</div>
|
|
122
|
+
<div style={{ position: 'absolute', left: 0, top: 125, width: '100%', textAlign: 'center', opacity: visible(t, 3, 6.7) }}><span style={{ fontSize: 76, fontWeight: 650, letterSpacing: -3 }}>把素材,编成故事。</span></div>
|
|
123
|
+
<Heading t={t} start={10.2} end={14.1} label="ONE TIMELINE">{'画面、声音,\n在同一条线上。'}</Heading>
|
|
124
|
+
<Heading t={t} start={14} end={17.9} label="PICTURE × SOUND">{'每个落点,\n都有回响。'}</Heading>
|
|
125
|
+
<Heading t={t} start={18} end={22} label="REVIEW → DELIVER">{'交付之前,\n看清每一帧。'}</Heading>
|
|
126
|
+
<div style={{ position: 'absolute', left: 116, top: 590, color: '#62708a', fontSize: 29, lineHeight: 1.8, opacity: layers }}>画面、文字和声音分层编排。<br />同一事件时间,驱动不同轨道。</div>
|
|
127
|
+
<div style={{ position: 'absolute', inset: 0, perspective: 1800 }}>
|
|
128
|
+
{[-1, 1].map((side, i) => <div key={side} style={{ position: 'absolute', left: 960 + side * (600 + (1 - move(t, 2.9, 4.2)) * 300) - 260, top: 390 + i * 46, width: 520, height: 340, borderRadius: 24, overflow: 'hidden', opacity: fan, transform: `rotateY(${-side * 24}deg) rotateZ(${side * 9}deg)`, boxShadow: '0 30px 70px #16366b24' }}><Artwork t={t + i} variant={i + 1} /></div>)}
|
|
129
|
+
{[2, 1].map((i) => <div key={i} style={{ position: 'absolute', left: pose.x - pose.w / 2, top: pose.y - pose.h / 2 + 94 * i, width: pose.w, height: pose.h, background: i === 1 ? '#e3edff' : '#b5dce1', border: `2px solid ${i === 1 ? '#98b4f0' : '#72b6bf'}`, borderRadius: 26, opacity: layers * 0.92, transform: `rotateX(${pose.rx}deg) rotateY(${pose.ry}deg) rotateZ(${pose.rz}deg)`, boxShadow: '0 24px 50px #14356f15' }}>
|
|
130
|
+
<div style={{ position: 'absolute', bottom: 16, left: 26, fontSize: 23, color: '#3d5e90' }}>{i === 1 ? 'TEXT / 文字层' : 'AUDIO / 声音层'}</div>
|
|
131
|
+
{i === 2 && <svg viewBox="0 0 900 200" style={{ width: '100%', height: '50%', position: 'absolute', bottom: 24 }}>{Array.from({ length: 75 }, (_, j) => { const h = 14 + Math.abs(Math.sin(j * 0.8 + t * 3)) * 100; return <rect key={j} x={j * 12} y={100 - h / 2} width="6" height={h} rx="3" fill="#5999a5" opacity="0.7" />; })}</svg>}
|
|
132
|
+
</div>)}
|
|
133
|
+
<div style={{ position: 'absolute', left: pose.x - pose.w / 2, top: pose.y - pose.h / 2, width: pose.w, height: pose.h, borderRadius: radius, overflow: 'hidden', background: blue, transform: `rotateX(${pose.rx}deg) rotateY(${pose.ry}deg) rotateZ(${pose.rz}deg)`, boxShadow: full > 0.99 ? 'none' : '0 38px 100px #102a5d30' }}>
|
|
134
|
+
<div style={{ position: 'absolute', inset: 0, opacity: 1 - close }}><Artwork t={t} labelOpacity={1 - move(t, 6.15, 6.45) * (1 - move(t, 10.7, 11))} /></div>
|
|
135
|
+
<div style={{ position: 'absolute', left: 108, top: 330, color: '#fff', fontSize: 100, fontWeight: 650, letterSpacing: -5, lineHeight: 1.2, opacity: full }}>{'让每一镜'}<br />{'有下一镜。'}</div>
|
|
136
|
+
<div style={{ position: 'absolute', left: 112, bottom: 95, color: '#a5cdfd', font: '22px monospace', letterSpacing: 3, opacity: visible(t, 7.3, 9.7) }}>ONE OBJECT. ONE CONTINUOUS WORLD.</div>
|
|
137
|
+
<div style={{ position: 'absolute', inset: 0, display: 'flex', alignItems: 'center', justifyContent: 'center', opacity: close }}><svg viewBox="0 0 100 100" width="82" height="82"><path d="M36 20L78 50L36 80Z" fill="#fff" /></svg></div>
|
|
138
|
+
</div>
|
|
139
|
+
</div>
|
|
140
|
+
<Timeline t={t} />
|
|
141
|
+
<Review t={t} />
|
|
142
|
+
<div style={{ position: 'absolute', left: 0, top: 520, width: '100%', textAlign: 'center', opacity: move(t, 22.2, 23.2), transform: `translateY(${(1 - move(t, 22.2, 23.2)) * 32}px)` }}>
|
|
143
|
+
<div style={{ fontSize: 88, fontWeight: 650, letterSpacing: -4 }}>从想法,到成片。</div>
|
|
144
|
+
<div style={{ fontFamily: 'monospace', fontSize: 37, color: blue, marginTop: 30 }}>pi-video-gen</div>
|
|
145
|
+
<div style={{ fontSize: 28, color: '#6c7c96', marginTop: 55, letterSpacing: 5 }}>生成 · 编排 · 审阅 · 交付</div>
|
|
146
|
+
</div>
|
|
147
|
+
<Audio src={staticFile('score.wav')} volume={1} />
|
|
148
|
+
</AbsoluteFill>;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
registerRoot(() => <Composition id="ContinuousDemo" component={Demo} width={1920} height={1080} fps={30} durationInFrames={durationSec * 30} />);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export const durationSec = 26;
|
|
2
|
+
export const events = { gather: 4.2, enter: 7.6, edit: 11, layers: 14.8, review: 18.5, resolve: 22.8 };
|
|
3
|
+
// One persistent preview travels through the whole film. Positions are composition pixels.
|
|
4
|
+
export const poses = [
|
|
5
|
+
{ at: 0, x: 1390, y: 565, w: 660, h: 550, rx: -8, ry: -24, rz: 8 },
|
|
6
|
+
{ at: 2.5, x: 1390, y: 565, w: 660, h: 550, rx: -8, ry: -24, rz: 8 },
|
|
7
|
+
{ at: events.gather, x: 960, y: 580, w: 1100, h: 620, rx: 0, ry: 0, rz: 0 },
|
|
8
|
+
{ at: 6.2, x: 960, y: 580, w: 1100, h: 620, rx: 0, ry: 0, rz: 0 },
|
|
9
|
+
{ at: events.enter, x: 960, y: 540, w: 1920, h: 1080, rx: 0, ry: 0, rz: 0 },
|
|
10
|
+
{ at: 9.5, x: 960, y: 540, w: 1920, h: 1080, rx: 0, ry: 0, rz: 0 },
|
|
11
|
+
{ at: events.edit, x: 1210, y: 410, w: 1020, h: 574, rx: 0, ry: 0, rz: 0 },
|
|
12
|
+
{ at: 13.6, x: 1210, y: 410, w: 1020, h: 574, rx: 0, ry: 0, rz: 0 },
|
|
13
|
+
{ at: events.layers, x: 1260, y: 410, w: 940, h: 530, rx: 34, ry: -14, rz: -8 },
|
|
14
|
+
{ at: 16.8, x: 1260, y: 410, w: 940, h: 530, rx: 34, ry: -14, rz: -8 },
|
|
15
|
+
{ at: events.review, x: 1260, y: 410, w: 960, h: 540, rx: 0, ry: 0, rz: 0 },
|
|
16
|
+
{ at: 21.2, x: 1260, y: 410, w: 960, h: 540, rx: 0, ry: 0, rz: 0 },
|
|
17
|
+
{ at: events.resolve, x: 960, y: 360, w: 146, h: 146, rx: 0, ry: 0, rz: 0 },
|
|
18
|
+
{ at: durationSec, x: 960, y: 360, w: 146, h: 146, rx: 0, ry: 0, rz: 0 },
|
|
19
|
+
];
|
|
@@ -12,7 +12,25 @@ When visual direction is genuinely open, outline up to three directions that dif
|
|
|
12
12
|
|
|
13
13
|
**Ready to proceed:** the central claim has evidence, the material and visual direction support it, and any decision that blocks faithful production is resolved.
|
|
14
14
|
|
|
15
|
-
## 2.
|
|
15
|
+
## 2. Choose a motion language that serves the explanation
|
|
16
|
+
|
|
17
|
+
Before choosing effects, identify what the viewer needs to understand. Record a primary motion language and why the actual material supports it:
|
|
18
|
+
|
|
19
|
+
| Viewer needs | Useful motion language | Material and acceptance |
|
|
20
|
+
|---|---|---|
|
|
21
|
+
| Follow an operation from request to result | Cursor-led demonstration or a continuous state change | Real interface, CLI output or faithful reconstruction; the operation and its result remain identifiable |
|
|
22
|
+
| Understand how parts work together | Mechanism animation or a system diagram | Actual components and relationships; motion shows a connection, transformation or dependency |
|
|
23
|
+
| See evidence or a comparison | Data-led animation or editorial annotation | Sourced values and readable comparisons; the important result gets reading time |
|
|
24
|
+
| Experience the range of a creative tool | One deeply developed artifact or a varied showcase with a shared visual anchor | Actual outputs or clearly labeled original studies; each passage demonstrates a different capability |
|
|
25
|
+
| Remember a short claim or brand | Kinetic typography, a crafted object or a generative composition | The product's own words, identity and visual material; the picture supports the claim |
|
|
26
|
+
|
|
27
|
+
Combine languages only where the change helps the message. Depth and a continuous camera are choices, not default quality upgrades. A coherent palette can support very different framing, density and movement; visual unity does not require repeating one layout.
|
|
28
|
+
|
|
29
|
+
For capability showcases, name what each passage proves. Recoloring the same object or moving the same card through several arrangements does not establish range. For product demonstrations, use the real tool/interface/output rather than an invented editor. Label workflow illustrations as illustrations and keep them distinct from evidence of shipped functionality. A persistent decorative object earns its place only when its transformations explain something; keep a shared color, cursor or framing anchor instead when the subject needs to change.
|
|
30
|
+
|
|
31
|
+
**Ready to plan:** the chosen language fits the requested film, its hero moment has suitable material, and the film would lose something meaningful if another product's name were substituted.
|
|
32
|
+
|
|
33
|
+
## 3. Design the rhythm before building every scene
|
|
16
34
|
|
|
17
35
|
Choose the route that carries the explanation: continuous movement through one world, a montage linked by a beat or visual anchor, or a continuous spine with a short montage burst. For a demo, preserve causal continuity across UI/device changes: the same request, object or person should remain identifiable through the handoff.
|
|
18
36
|
|
|
@@ -27,37 +45,48 @@ Write a beat map in chat or the treatment. Each row describes what changes for t
|
|
|
27
45
|
|
|
28
46
|
This is an example, not a fixed structure or timing requirement. Give each shot a verb that explains something: connect, filter, compare, reveal, assemble. Vary speed, scale, density and stillness to serve the message. For an energetic launch, consider an early hook, a denser passage and a contrasting pause before the hero moment. A quiet explainer can use a different curve. A planned reading hold is useful; repeated motion that adds no information is a revision candidate.
|
|
29
47
|
|
|
48
|
+
Before filling every beat, describe the energy of each stretch and what changes between them: shot scale, movement speed, camera behavior, information density or musical texture. For an energetic launch or showcase, start with a strong subject already visible and an early meaningful action, build to a visibly denser passage, then leave space for the hero result and ending. A continuous demonstration can vary detail, scale and reading time without cuts. A hybrid film can insert a short, readable burst of different materials around a stable anchor. Choose the route from the brief; a fixed cut count or one-take rule is not an acceptance criterion.
|
|
49
|
+
|
|
50
|
+
Review the proposed curve for repeated grammar: several seconds of gentle enlargement, a hold, then a shrink, repeated throughout, remains flat even when the total duration or scene count increases. Replace repetition with a change that communicates new information. In a planned burst, keep the focal area trackable and align intentional cut hits to the actual music grid; allow explanatory passages to breathe. Do not label a background loop with timed accents a complete musical arc unless its sections actually change with the film.
|
|
51
|
+
|
|
30
52
|
Choose the typography and palette from the actual brand, product or source material. Keep numbers and essential text editable and render exact UI, charts and text through Timeline overlays or Remotion as appropriate. Use movement to bring the eye to a reading area, let it settle, then allow the state change or reading hold. Check the transition between scenes at normal speed, especially when the viewer must follow something across screens.
|
|
31
53
|
|
|
54
|
+
For continuous or hybrid films, read [motion authoring](motion-authoring.md). For each important handoff, name the on-screen object that persists, its state before and after, and the time the viewer should recognize the result. “Fade into the next card” is insufficient when the explanation depends on following the same request or object. A deliberate montage cut remains valid; record its purpose rather than inventing a continuity link. Do not force a continuous camera onto supplied footage or a story that benefits from cuts.
|
|
55
|
+
|
|
32
56
|
**Ready to build:** every scene adds identifiable information, every essential claim has a readable landing/hold, and the film's pace supports the intended tone.
|
|
33
57
|
|
|
34
|
-
##
|
|
58
|
+
## 4. Prove the hardest moment, then assemble
|
|
35
59
|
|
|
36
60
|
Build the shot or transition most likely to fail first: a dense UI view, cross-device handoff, difficult chart, or hero reveal. Inspect it at delivery resolution using the Remotion project's native preview/still renderer, or source-frame and overlay stills for Timeline, before extending the same treatment to the full film. Correct unreadable text, collisions, unclear focus and broken continuity at this stage. Reserve `video_compose` or `video_render` for one complete initial film export after all scenes are ready; a full-length placeholder film is not a required proof stage. If a separate prototype video is genuinely necessary, explain that extra render explicitly rather than silently adding it to the default workflow.
|
|
37
61
|
|
|
38
62
|
For a multi-scene film, inspect representative key frames for the opening, hero moment, densest information and ending. These may be frames from the same prototype rather than separate generated images. For missing mascot portraits or shot frames, use `image_generate` through the pi-image-gen skill and record the returned asset paths. Keep supplied screenshots as original assets. Then build the remaining scenes using the chosen Timeline or Remotion path and any generated shots the film actually needs.
|
|
39
63
|
|
|
40
|
-
##
|
|
64
|
+
## 5. Plan sound on the same clock as picture
|
|
41
65
|
|
|
42
66
|
For each important sound cue, record the event and landing time or frame. Put a click on the click, a hit on the actual landing, and a reveal cue on the reveal; the start of a tween is not necessarily its landing. Use a shared beat grid when the edit is music-driven. After changing timing, update the affected picture and sound cues together.
|
|
43
67
|
|
|
44
68
|
Use existing or appropriately licensed audio first. For narration-led films, keep speech intelligible and give important words space. If word-level timing is available, record its source and precision before anchoring a visual to it. Sentence timestamps and interpolated word estimates MUST NOT be described as verified word alignment.
|
|
45
69
|
|
|
70
|
+
Choose the musical character from the product and visual direction before choosing instruments: for example, interlocking acoustic pulses for a systems explanation, a restrained electronic groove for a technical launch, or a warm small ensemble for a human story. These are options, not genre defaults. Describe the opening, build, peak and resolution in the treatment, including changes in instrumentation, density and silence. Reuse the film's motif at the important reveal or brand ending. A local synthesizer is one implementation option; prefer suitable existing audio when it better serves the intended sound. Additional oscillators, a louder master or more accents alone do not demonstrate better composition.
|
|
71
|
+
|
|
46
72
|
Implement only what the selected path supports:
|
|
47
73
|
|
|
48
74
|
- **Timeline** supports BGM, per-segment narration, source audio and sentence-timed subtitles when Edge TTS supplies boundaries. Its BGM uses a fixed mix level; it does not dynamically duck under speech or accept arbitrary timed sound-effect/word-cue fields. Keep its input to the documented schema.
|
|
49
75
|
- **Remotion** can place existing audio assets and animate against frame/time values in the editable project. Author sound timing and volume changes there when the film needs them, following the Remotion handoff. Generated media is still handed off through the existing asset mechanism.
|
|
50
76
|
- A sound plan alone creates no audio. If suitable audio or the means to produce the requested treatment is missing, state that gap and resolve the choice with the user. Promise local composition or word alignment only after an available implementation has been verified.
|
|
51
77
|
|
|
52
|
-
##
|
|
78
|
+
## 6. Review the exported film and handle revisions
|
|
53
79
|
|
|
54
80
|
Review the actual final export against the claim and beat map, not just the source preview or a successful render result. Use the existing QC artifacts and tools available in the environment; an additional QA command is not assumed to exist.
|
|
55
81
|
|
|
56
82
|
1. **Stills:** inspect the key moments at delivery resolution for accurate UI/data/text, readable type, crop, overlap, contrast and a clear focal area. Check frames immediately after scene changes as well as settled frames; an early flash can disappear before a segment's midpoint.
|
|
83
|
+
For Remotion, put selected handoff times into the spec's `qcTimesSec`: include before, during and after a risky transition, not only its attractive final state. Check the carrier and essential text remain in frame. The returned report binds these samples to the actual exported file's SHA-256; it does not measure semantic continuity.
|
|
57
84
|
2. **Motion:** watch the full export at normal speed when playback inspection is available. Check whether a viewer can follow causal changes, read each essential state, and feel the intended pace. Record timestamped problems, including unintended flashes, idle stretches and abrupt joins. Planned holds and cuts need judgment rather than automatic rejection.
|
|
58
85
|
3. **Audio:** listen when audio inspection is available. Check voice clarity, cue synchronization, balance, clipping and the ending. Waveforms, spectra and loudness measurements are supporting evidence; they do not establish that the music sounds good or speech is intelligible.
|
|
59
86
|
4. **Delivery:** report the exact exported path/version, what passed visual/motion/audio review, any unreviewed modality, and accepted limitations. A render/QC pass MUST NOT be presented as aesthetic or user acceptance. Preserve the editable source and asset references with the deliverable.
|
|
60
87
|
|
|
61
88
|
When playback or listening is unavailable, inspect the frames and timing/audio evidence that are available, identify the remaining viewing/listening check, and report the export with that limitation. Do not claim a normal-speed watch or listening pass that did not happen.
|
|
62
89
|
|
|
90
|
+
Before presenting an initial creative film, write up to three concrete director's notes against the actual version: its weakest opening/hero choice, a passage whose material or movement adds little, and a pacing, typography or sound problem. Give the timestamp and intended improvement, then address the supported findings. Use fewer notes when fewer real issues exist; do not manufacture defects or numeric scores. A low-information passage needs a content or direction decision rather than simply more movement. Check that the main claim survives muted playback at the intended screen size. Report any unresolved issue or unreviewed modality honestly; technical QC alone does not close these notes.
|
|
91
|
+
|
|
63
92
|
For feedback, bind the request to an exact exported version/path (and hash when available), timestamp, screenshot or screen location, and expected change. Inspect that moment before editing; ask only when the intended change cannot be inferred. Make the smallest relevant source change, update affected timing, and create a new job for revised inputs under the existing immutable-job rules. Review the target window, its neighboring joins and any shifted downstream cues; deliver the new version with a concrete account of what changed.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Motion authoring
|
|
2
|
+
|
|
3
|
+
Use this guide for precise UI, text and chart animation in an editable Remotion project. It supplements film direction; a single generated clip, straightforward assembly, supplied edit or intentional montage does not need a continuous-motion plan. No external skill or animation runtime is required.
|
|
4
|
+
|
|
5
|
+
## Make each handoff explain a change
|
|
6
|
+
|
|
7
|
+
Start with the real product evidence. Choose the object the viewer follows: a request, selection, data point or physical item. Write only the important handoffs:
|
|
8
|
+
|
|
9
|
+
| Time window | Persistent object | Before → after | Viewer focus and reading hold | Sound landing |
|
|
10
|
+
|---|---|---|---|---|
|
|
11
|
+
| 1–3s | Request composer | Compact input → expanded product workspace | Keep the entered request readable while its container grows | Expansion settles at 3s |
|
|
12
|
+
| 5–6s | Same request row | Submitted request → visible result | Follow the selected row; hold the result for 2s | Result becomes visible at 6s |
|
|
13
|
+
| 9–10s | Result mark | Product result → closing brand anchor | Keep a single focal area and enough time for the closing text | Resolve at the landing, then leave room for the tail |
|
|
14
|
+
|
|
15
|
+
These are illustrative times, not a required structure. Use a persistent object only when it clarifies causality. For a deliberate cut, record why the new view helps. Different shot lengths, pauses and speed changes should serve information rather than satisfy a numerical rhythm threshold.
|
|
16
|
+
|
|
17
|
+
## Build on a shared event clock
|
|
18
|
+
|
|
19
|
+
Record the important landing frames in one project-level object, such as `events = { expanded: 90, result: 180 }`. Derive movement start frames from those landings and the chosen movement duration. Let both the picture and audio reference those values. Changes to a landing then move the related sound with it.
|
|
20
|
+
|
|
21
|
+
Use one visible carrier across the transition, interpolating position, bounds and relevant state. Keep animation a function of the current frame; random choices must be seeded. Reuse Remotion's motion helpers. Do not reset the carrier by swapping independently animated scene trees halfway through its handoff. Camera movement should preserve the carrier and important text within the delivery frame; settle the camera before asking the viewer to read dense content. Check a vertical layout independently rather than cropping a horizontal one.
|
|
22
|
+
|
|
23
|
+
For sound, use supplied or appropriately licensed assets. Put sparse accents on meaningful results, not on every animated property. Music should leave room for speech and important effects; author gain changes in the project when needed. Word-triggered animation requires timing with a known source and precision. Sentence boundaries or interpolated estimates MUST NOT be described as measured word alignment. Missing sound assets remain a production gap, not evidence of a completed sound treatment.
|
|
24
|
+
|
|
25
|
+
## Review the vulnerable moments
|
|
26
|
+
|
|
27
|
+
For each risky handoff, choose a small set of before, middle and after timestamps. Include the moment of greatest zoom or travel, dense text and the result hold. Put up to 32 selected seconds-based times into `qcTimesSec` in the existing Remotion spec. Batch work still supplies `videoGenShotTimesSec` for every shot. The checks validate times and extract actual export frames; they do not recognize the carrier or judge aesthetics.
|
|
28
|
+
|
|
29
|
+
Inspect those returned frames for lost objects, crop, unreadable text, duplicated elements and a focus jump. Watch the actual transition at normal speed when playback inspection is available. Listen to the export when audio inspection is available. A still cannot prove temporal continuity, and a peak/loudness measurement cannot prove pleasant or intelligible sound. State unreviewed modalities explicitly.
|
|
30
|
+
|
|
31
|
+
For a revision, identify the exported version/path, timestamp, screen region and expected change. Edit the smallest relevant source section, update shared event timing, render into a new job and review both the changed window and neighboring joins. Keep the editable project, source assets and exported version together as the delivery record.
|
|
@@ -18,4 +18,16 @@ For batch `video_render`, the selected composition must expose `props.videoGenSh
|
|
|
18
18
|
|
|
19
19
|
The final MP4 and assembly manifest live under `<jobDir>/assembly/` for `video_render`; direct `video_compose` writes them in its own job directory. Preserve the Remotion project as an editable deliverable. Inspect actual frames after rendering; a successful bundle and media probe do not establish visual correctness.
|
|
20
20
|
|
|
21
|
+
## Review moments and sound
|
|
22
|
+
|
|
23
|
+
`remotion-input.json` and `assembly.type: "remotion"` accept optional `qcTimesSec`, an array of at most 32 finite, non-negative times in seconds. Each must be strictly less than the selected composition duration; batch preflight checks this before model submission, and the export is checked again. For a handoff at 4 seconds, for example, use `"qcTimesSec": [3.8, 4, 4.2]`. See [motion authoring](motion-authoring.md) for how to select these times.
|
|
24
|
+
|
|
25
|
+
New renders automatically extract beginning, middle and ending samples from the encoded MP4, plus the requested times, sorted and deduplicated. They return `qcFrames` and `qcReportPath`; the report records video-track duration, sample times and final video SHA-256. Batch `video_render` includes these frames alongside its shot samples; its QC report links the Remotion frame list and SHA-bound report under `remotion`. Frame/report hashes join the Remotion manifest and are verified on resume. Missing or changed evidence fails instead of silently regenerating it. Historical manifests without QC hashes resume with no recorded QC; create a new job to obtain evidence. `qcStatus: "recorded"` means evidence exists, not that the film passed visual, motion or audio review.
|
|
26
|
+
|
|
27
|
+
Keep `videoGenShotTimesSec` for the existing batch shot-visibility contract. `qcTimesSec` supplements that contract with handoff and reading checks and does not replace it. The plugin does not inspect object identity or automatically assign a continuity score.
|
|
28
|
+
|
|
29
|
+
For frame-derived animation, persistent carriers and shared picture/sound timing, follow [motion authoring](motion-authoring.md). Existing audio can be passed through `assets` and read with `staticFile(videoGenAssets[id])`; author gain changes and clip placement in the project. This does not add timed audio or dynamic ducking to FFmpeg Timeline.
|
|
30
|
+
|
|
31
|
+
The [continuous motion film](../examples/continuous-demo/README.md) is an editable 26-second project: one persistent preview travels through material collection, a full-frame shot, an editing timeline, separated picture/text/audio planes, a review view and a closing play mark. Shared event times drive both motion landings and the locally synthesized score. Its UI illustrates the workflow rather than an editor supplied by the plugin.
|
|
32
|
+
|
|
21
33
|
Remotion has its own [license terms and FAQ](https://www.remotion.dev/docs/license/faq). The project owner should check the applicable terms before using this path in commercial automation. The plugin does not bundle Remotion code.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Seedance trusted portraits and provider-managed assets
|
|
2
|
+
|
|
3
|
+
Before any paid Seedance call that may contain a recognizable human face:
|
|
4
|
+
|
|
5
|
+
1. **Classify the source.** Ordinary uploaded or generated images/videos of a recognizable real person may be rejected by Seedance's privacy checks. Do not treat a local file path or public URL as an authorized portrait.
|
|
6
|
+
2. **Search preset personas when needed.** If the user has not already chosen an identity, run `<skillDir>/scripts/search-seedance-personas.mjs` with `--query "<space-separated traits>" --framing half|full --limit 5`. Use `half` for close/medium portrait shots and `full` when the whole body or body movement must be visible. When the user has not selected or delegated the identity choice, present the bounded matches with label, short bio, framing, and Asset ID for selection before a paid call. When that choice is delegated, select a suitable match and include the exact asset in the paid scope. The script is the access path; do not load the entire 3.6 MB JSON catalog into context.
|
|
7
|
+
3. **Use the exact trusted asset.** For a catalog choice, copy the returned `selectedAssetId`. Otherwise ask the user for a preset-avatar or Active authorized-person `asset-...` ID from the current Ark account/project. Catalog IDs were observed on 2026-08-24 and are not documented as permanent or cross-account. If the active account rejects one, ask the user to copy the current ID from that account's virtual-avatar library.
|
|
8
|
+
4. **Preserve modality order.** Put provider assets in `referenceAssets` as `{ "modality": "image|video|audio", "assetId": "asset-..." }`. Order is significant within each modality. In the prompt say `Image 1`, `Video 1`, or `Audio 1`; never expose or cite the Asset ID in prompt prose. For the built-in Seedance 2.0 models, keep each request within 9 image references total (local frames plus image assets), 3 video assets, and 3 audio assets.
|
|
9
|
+
5. **Reconfirm the paid context.** Approval is scoped to the exact asset list, provider/account context, model, clip count, and duration. An unchanged resumed job retains its existing approval. A changed asset, account/project, provider, model, or job requires a new confirmation.
|
|
10
|
+
6. **Keep onboarding out of scope.** This package submits already-created assets. It does not perform identity verification, authorization H5 flows, asset activation, upload, or asset-library management.
|
|
11
|
+
|
|
12
|
+
Official references: [preset avatars](https://docs.volcengine.com/docs/82379/2608626?lang=zh#preset-avatar), [authorized-person assets](https://docs.volcengine.com/docs/82379/2223965?lang=zh), and [Seedance asset request format](https://console.volcengine.com/ark/region:cn-beijing/docs/82379/2333589?projectName=default&lang=zh#d9a7d853).
|
|
13
|
+
|
|
14
|
+
## Public image/video/audio materials
|
|
15
|
+
|
|
16
|
+
When the user asks for built-in Seedance materials, action references, camera references, visual styles, environments, characters, or sample voices, read [`seedance-public-material-library.md`](seedance-public-material-library.md) completely before proposing choices. Use the exact Chinese display labels from that catalog, copy its exact Asset IDs into `referenceAssets`, and keep the selected media in modality order. These IDs were read from the public material cards, not from the separate virtual-avatar library. If the active account rejects a listed ID, re-open the experience center and copy the current card ID instead of guessing or substituting a media URL.
|
|
@@ -6,7 +6,7 @@ Observed on 2026-08-24. The Asset IDs below were copied directly from each
|
|
|
6
6
|
material card's **ID** button; they were not inferred from labels or media URLs.
|
|
7
7
|
The catalog may change independently of the API. It currently contains 71
|
|
8
8
|
images, 36 videos, and 80 audio samples. The separate **虚拟人像库** is not
|
|
9
|
-
duplicated here; search it through the persona workflow in [trusted portraits](
|
|
9
|
+
duplicated here; search it through the persona workflow in [trusted portraits](seedance-assets.md).
|
|
10
10
|
|
|
11
11
|
## Selection contract
|
|
12
12
|
|
|
@@ -24,7 +24,7 @@ duplicated here; search it through the persona workflow in [trusted portraits](.
|
|
|
24
24
|
5. Preserve modality order. In prompt prose refer to attachments as `Image 1`,
|
|
25
25
|
`Video 1`, or `Audio 1`, never by Asset ID.
|
|
26
26
|
6. Do not reuse these public-material IDs as preset-avatar IDs. For a
|
|
27
|
-
recognizable real person, follow [trusted portraits](
|
|
27
|
+
recognizable real person, follow [trusted portraits](seedance-assets.md) and obtain a current-account trusted
|
|
28
28
|
portrait or preset-avatar Asset ID from the user.
|
|
29
29
|
|
|
30
30
|
Example:
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Shot planning and frame preparation
|
|
2
|
+
|
|
3
|
+
Use a shot book when narrative continuity benefits from one. Author it in conversation; save `project.json` beside the editable source, outside generated job directories, when the project needs an editable planning record.
|
|
4
|
+
|
|
5
|
+
```jsonc
|
|
6
|
+
{
|
|
7
|
+
"title": "...", "style": "Cartoon",
|
|
8
|
+
"characters": [{ "id": "alice", "visible": true,
|
|
9
|
+
"appearance": "long blonde hair, blue eyes, slender", // static features
|
|
10
|
+
"outfit": "red scarf, black leather jacket" }], // dynamic features
|
|
11
|
+
"shots": [{
|
|
12
|
+
"id": "s1",
|
|
13
|
+
"intent": "Wide shot, rainy alley. <Alice> enters from the left, stops under the streetlamp…",
|
|
14
|
+
"scene": "Rainy alley at night, neon signs, wet pavement", // optional: the shot's setting
|
|
15
|
+
"firstFrame": "…pure static description of the FIRST frame…",
|
|
16
|
+
"lastFrame": "…(optional) pure static description of the LAST frame…",
|
|
17
|
+
"visuals": "Static camera, wide shot from across the street…", // camera + framing only
|
|
18
|
+
"action": "A woman with long blonde hair and a red scarf walks in from the left…",
|
|
19
|
+
"effects": "…(optional) time-varying visuals: rain picks up, neon reflections intensify…",
|
|
20
|
+
"audio": "[Sound Effect] rain, distant traffic. [Speaker] Alice (soft): \"We're here.\"",
|
|
21
|
+
"visibleCharacters": ["alice"],
|
|
22
|
+
"durationSec": 5,
|
|
23
|
+
"continuityGroup": "alley",
|
|
24
|
+
"startFrameFromShotId": "s0", // optional: this shot's frame builds on s0's frame
|
|
25
|
+
"continuityNote": "In s0's frame Alice faces away; front view missing"
|
|
26
|
+
}]
|
|
27
|
+
}
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Field rules:
|
|
31
|
+
|
|
32
|
+
- **Give each story shot a purpose** (establish / emotion / reaction). Choose framing to serve that purpose; use close-ups for emotion and wide shots for context when appropriate.
|
|
33
|
+
- **Keep dialogue within the selected model's duration and audio support.** Character names in `intent` may be wrapped in angle brackets: `<Alice>`.
|
|
34
|
+
- **firstFrame / lastFrame are pure static snapshots** — no ongoing actions ("he is sitting, leaning forward", NOT "he is about to stand"). Include shot size, angle, composition, who is where and facing which way.
|
|
35
|
+
- **visuals = camera + framing only** (movement, shot size, angle, focus); **action = in-frame movement only**. Split them — they become separate labeled sections in the assembled prompt. Refer to characters by visible traits ("the woman in the red scarf"), never by name.
|
|
36
|
+
- **scene is the shot's setting**, copied verbatim into the render spec's `prompt.scene`. Optional when the first frame fully anchors the setting; write it when the setting carries mood/lighting the frame may not convey.
|
|
37
|
+
- **effects is for what a static frame cannot carry**: transformations, lighting/atmosphere shifts, particles, slow motion. Omit when the shot is visually static.
|
|
38
|
+
- **lastFrame needed when**: composition/focus changes drastically, a character enters or turns to face camera, a major reveal happens. Otherwise omit it.
|
|
39
|
+
- **Use `continuityGroup` where shots share a space or visual base.** Start a new group when the setting or camera relationship warrants it.
|
|
40
|
+
- **continuityGroup** = shots sharing a space/base image; **startFrameFromShotId** pins a specific parent frame for composition; **continuityNote** says what the parent frame lacks (the frame prompt must then keep the background and replace those elements). Self-check: parent shot EXISTS, comes EARLIER, same continuityGroup, no cycles.
|
|
41
|
+
- **audio** uses `[Sound Effect] …` / `[Speaker] Name (Emotion): "line"` format.
|
|
42
|
+
- **durationSec and all capability values come from `video_capabilities`** — never from memory or this document. Durations, resolutions, ratios, audio and frame support differ per model and change over time.
|
|
43
|
+
- **Behavioral quirks worth knowing** (still verify with `video_capabilities`): some models have no native audio (omit audio cues or the render is silent); some cannot do last-frame interpolation (never pass lastFrame to them); HappyHorse takes a first frame OR reference images in one call, not both — cite references in the prompt as `[Image 1]`, `[Image 2]`, …
|
|
44
|
+
|
|
45
|
+
## Frame preparation (via `image_generate`)
|
|
46
|
+
|
|
47
|
+
When a shot book needs missing character portraits or first/last frames, call `image_generate` to create those images before `video_render`. Follow the **pi-image-gen skill** for parameters, sizes, `n`, and edit labeling; it is the single authority. Two video-specific handoff rules:
|
|
48
|
+
|
|
49
|
+
- **Never assume a saved filename**: the actual extension follows the MIME type and collisions get `-v2`. **The returned absolute path is the only truth** — record it immediately in `assets.json` (see below) and reference it in the render spec.
|
|
50
|
+
- `assets.json` in the job dir: `{ "assets": { "<shotId>/<part>": { "sourcePath": "…" } } }` mapping semantic assets (e.g. `s1/firstFrame`, `alice/front`) to real paths.
|
|
51
|
+
|
|
52
|
+
**Character portrait views (generate only those the planned frames need)**:
|
|
53
|
+
|
|
54
|
+
- front (text-to-image): `Generate a full-body, front-view portrait of character {identifier} based on the following description, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. Gazing straight ahead. Standing with arms relaxed at sides. Natural expression. Features: {appearance}; {outfit}. Style: {style}`
|
|
55
|
+
- side (edit with front as reference): `Generate a full-body, side-view portrait of character {identifier} based on the provided front-view portrait, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. Facing left. Standing with arms relaxed at sides.`
|
|
56
|
+
- back (edit with front as reference): `Generate a full-body, back-view portrait of character {identifier} based on the provided front-view portrait, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. No facial features should be visible.`
|
|
57
|
+
|
|
58
|
+
If an optional side/back view fails after one retry, reuse an existing suitable view. Characters with `visible: false` get no portraits.
|
|
59
|
+
|
|
60
|
+
**Reference selection for frames**: candidates = portraits of visible characters (ONE view each, chosen by facing) + continuity frames. Pick a SMALL set of the most relevant ones — same camera/group first, most recent frames first, drop redundant near-duplicates, prefer the portrait when a character newly appears. How many images a call accepts is pi-image-gen's authority (its skill/tool description), not this document's.
|
|
61
|
+
|
|
62
|
+
**Frame prompt assembly**: prefix each reference image with its role, then the frame description mapping elements to images:
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
Image 1: A front view portrait of Alice.
|
|
66
|
+
Image 2: [alley] Wide shot of the rainy alley from shot s1.
|
|
67
|
+
Create an image based on the following description: <firstFrame text>. The alley background should reference Image 2; Alice's appearance should reference Image 1.
|
|
68
|
+
```
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Timeline composition
|
|
2
|
+
|
|
3
|
+
Use for promos/explainers from still images and existing clips. Media rendering is local and uses no paid video model. If narration is requested, disclose that the explicit `edge-tts:<voice>` option sends narration text to Microsoft before using it.
|
|
4
|
+
|
|
5
|
+
1. **Collect existing images/screenshots/clips first**, and use `image_generate` for missing or explicitly requested replacement visuals. Keep all source media outside the job directory, then author `<jobDir>/timeline-input.json`. `assets/`, `overlays/`, `audio/`, `segments/`, `qc/`, generated tracks, subtitles, and `final_video.mp4` are reserved pipeline outputs; a fresh job refuses any conflicts rather than deleting them.
|
|
6
|
+
```jsonc
|
|
7
|
+
{
|
|
8
|
+
"title": "产品宣传片",
|
|
9
|
+
"output": { "resolution": "1920x1080", "fps": 25, "codec": "h264" },
|
|
10
|
+
"voice": "edge-tts:zh-CN-YunyangNeural", // explicit opt-in: narration text is sent to Microsoft
|
|
11
|
+
"ttsFailureMode": "fail", // or "silent-subtitles" only after the user accepts that degradation
|
|
12
|
+
"subtitles": { "mode": "burn", "fontSize": 36,
|
|
13
|
+
"textColor": "#ffffff", "backgroundColor": "#000000", "backgroundOpacity": 0.55 },
|
|
14
|
+
"segments": [
|
|
15
|
+
{
|
|
16
|
+
"id": "intro",
|
|
17
|
+
"image": "/abs/frame-1.png",
|
|
18
|
+
"durationSec": 5, // image only: may be "auto" from narration
|
|
19
|
+
"motion": "kenburns-in", // image only
|
|
20
|
+
"transitionTo": { "type": "xfade", "style": "fade", "durationSec": 0.8 },
|
|
21
|
+
"overlay": { "title": "标题", "subtitle": "副标题", "position": "bottom-left" },
|
|
22
|
+
"narration": "这一段的中文旁白文本"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "demo",
|
|
26
|
+
"video": "/abs/demo.mp4",
|
|
27
|
+
"trimStartSec": 2.5,
|
|
28
|
+
"durationSec": 6, // video always uses a numeric duration
|
|
29
|
+
"fit": "contain", // contain | cover
|
|
30
|
+
"sourceAudio": { "muted": false, "volume": 0.25 }
|
|
31
|
+
}
|
|
32
|
+
]
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
2. **Chinese text NEVER comes from an image model** — titles/subtitles go in `overlay` and are rendered locally via SVG (no garbled CJK).
|
|
36
|
+
3. **Paid-model confirmation is unnecessary** for the local media render, but still show the segment count and total planned duration before calling `video_compose`. Obtain explicit agreement before sending narration text to Edge TTS.
|
|
37
|
+
4. Every segment contains exactly one of `image` or `video`. Video segments are normalized to the output resolution/fps, may be trimmed/scaled, and mix their source audio with narration before optional BGM. Video source audio without a stream degrades to silence; `sourceAudio.muted: true` or `volume: 0` disables it. A video's numeric `durationSec` is its fixed trim window; narration that does not fit is rejected instead of extending it.
|
|
38
|
+
5. Narration uses Edge TTS only with an explicit `edge-tts:<voice>` selection; its text is sent to Microsoft. Measured audio duration drives image `durationSec: "auto"`; subtitles use each segment's actual video timing. `subtitles.mode` defaults to `"soft"` (`mov_text`); `"burn"` renders the configured font/color/background directly into each narrated segment. TTS failures stop the job by default. Use `ttsFailureMode: "silent-subtitles"` only as an explicit degradation choice; it keeps the subtitle track and fills that segment with silence. Once accepted, that degradation is cached for the immutable job; create a NEW job to retry real narration.
|
|
39
|
+
6. On completion, review the QC frames in `<jobDir>/qc/` yourself (Read the PNGs) before showing the result — flipped/overlapping text only shows up visually. Soft `mov_text` subtitles are not burned into those PNGs; the pipeline separately verifies that the subtitle stream exists and that the SRT cues match the resolved segment timeline.
|
|
40
|
+
7. The spec is immutable per job: rerunning the same path resumes only regular job-local artifacts whose manifest hashes still match; changes require a NEW job directory. A committed artifact that is missing or changed is rejected rather than regenerated underneath cached downstream outputs. A completed manifest with any missing artifact hash is rejected; an interrupted manifest invalidates unverified downstream hashes before rebuilding an uncommitted upstream artifact.
|
|
@@ -21,3 +21,43 @@ Each shot has exactly one source. `videoPath` uses a local video and has no prom
|
|
|
21
21
|
Before the first paid submit, the script checks all shot references, existing media, model capabilities and assembly environment. Remotion additionally bundles the project with placeholder video, selects the composition and renders frame zero. This does not prove that the eventual generated codec or duration is compatible; downloaded clips are checked again. If assembly fails after generation, rerun the same script after fixing the environment. The saved shot files and handles remain; changing frozen project source, original public resources, existing assets or the script requires a new job.
|
|
22
22
|
|
|
23
23
|
Approval is scoped to the generated shots, provider/account, model and duration. Reuse an existing approval only for the same scope. On ambiguous remote submission, inspect the provider console, then use `/video-gen recover <jobId> <shotId> reset` only when the task is confirmed absent or `adopt <taskId>` when found. Cancellation stops local work and may not cancel provider billing. A legacy completed manifest without a saved final SHA-256 cannot verify the old film; compose its saved clips in a new job directory.
|
|
24
|
+
|
|
25
|
+
## Generated-shot example
|
|
26
|
+
|
|
27
|
+
Write `<outputDir>/<jobId>/render-input.json` (jobId: letters/digits/dash/underscore). This is a generated-shot example; for local `videoPath` shots and optional Timeline or Remotion assembly, use the mixed-source and assembly rules above:
|
|
28
|
+
|
|
29
|
+
```jsonc
|
|
30
|
+
{
|
|
31
|
+
"title": "…", "aspectRatio": "16:9",
|
|
32
|
+
"style": "Cartoon, warm palette, soft shading", // film-level look — from the shot book's style
|
|
33
|
+
"characters": [ // film-level registry — id + appearance/outfit merged
|
|
34
|
+
{ "id": "alice", "description": "long blonde hair, blue eyes, slender; red scarf, black leather jacket" }
|
|
35
|
+
],
|
|
36
|
+
"consistency": "Faces, hair and outfits stay identical across the shot, no morphing or drift.",
|
|
37
|
+
"negative": "no text, watermarks, or subtitles",
|
|
38
|
+
"shots": [{
|
|
39
|
+
"id": "s1",
|
|
40
|
+
"prompt": { // from the shot book's fields, verbatim
|
|
41
|
+
"scene": "Rainy alley at night, neon signs, wet pavement", // omit when the first frame says it all
|
|
42
|
+
"visuals": "Static camera, wide shot from across the street",
|
|
43
|
+
"action": "The woman in the red scarf walks in from the left, stops under the streetlamp",
|
|
44
|
+
"effects": "Rain picks up halfway through; neon reflections ripple in puddles",
|
|
45
|
+
"audio": "[Sound Effect] rain, distant traffic. [Speaker] Alice (soft): \"We're here.\"",
|
|
46
|
+
"visibleCharacters": ["alice"]
|
|
47
|
+
},
|
|
48
|
+
"firstFramePath": "<project>/path/from/assets.json.png", // optional for asset-only shots
|
|
49
|
+
"lastFramePath": "<project>/optional.png",
|
|
50
|
+
"referenceAssets": [
|
|
51
|
+
{ "modality": "image", "assetId": "asset-avatar-from-current-account" },
|
|
52
|
+
{ "modality": "audio", "assetId": "asset-voice-from-current-account" }
|
|
53
|
+
],
|
|
54
|
+
"durationSec": 5
|
|
55
|
+
}]
|
|
56
|
+
}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The plugin assembles each shot's labeled prompt (`[Style]` / `[Character]` / `[Scene]` / `[Visuals]` / `[Action]` / `[Effects]` / `[Audio]` + consistency and negative directives) — never pre-join a prompt string yourself. Validation fails before any paid call when: a generated shot has neither `firstFramePath` nor `referenceAssets`, `visuals`/`action` are empty, `visibleCharacters` references an id missing from `characters`, or a shot without `firstFramePath` lacks `style`/`scene`. Film-level `style`/`consistency`/`negative` apply to every shot — write them once; shot-level `scene` repeats per shot even when consecutive shots share a location (each shot is submitted independently).
|
|
60
|
+
|
|
61
|
+
Every reference-frame path that is present must resolve to a regular png/jpg/webp file inside the session cwd. Absolute paths are accepted only when they remain inside that approved project directory; symlinks and outside paths are rejected. `referenceAssets` are provider-managed and are not local files. Keep their order stable: changing an asset, modality, or order changes the request and requires a new job directory and paid confirmation.
|
|
62
|
+
|
|
63
|
+
Then call `video_render` with that path. Interrupted? Call it again with the same path — it resumes. If an ambiguous submit is reported, do not delete a shot or call render again blindly: run `/video-gen recover <jobId>`, check the provider console, then explicitly `reset` a confirmed-absent task or `adopt` its task id. Revisions? New job directory.
|