@vargai/sdk 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +23 -0
- package/.husky/README.md +102 -0
- package/.husky/commit-msg +9 -0
- package/.husky/pre-commit +12 -0
- package/.husky/pre-push +9 -0
- package/.size-limit.json +8 -0
- package/.test-hooks.ts +5 -0
- package/CONTRIBUTING.md +150 -0
- package/LICENSE.md +53 -0
- package/README.md +7 -0
- package/action/captions/index.ts +202 -12
- package/action/captions/tiktok.ts +538 -0
- package/action/cut/index.ts +119 -0
- package/action/fade/index.ts +116 -0
- package/action/merge/index.ts +177 -0
- package/action/remove/index.ts +184 -0
- package/action/split/index.ts +133 -0
- package/action/transition/index.ts +154 -0
- package/action/trim/index.ts +117 -0
- package/bun.lock +299 -8
- package/cli/index.ts +1 -1
- package/commitlint.config.js +22 -0
- package/index.ts +12 -0
- package/lib/ass.ts +547 -0
- package/lib/fal.ts +75 -1
- package/lib/ffmpeg.ts +400 -0
- package/lib/higgsfield/example.ts +22 -29
- package/lib/higgsfield/index.ts +3 -2
- package/lib/higgsfield/soul.ts +0 -5
- package/lib/remotion/SKILL.md +240 -21
- package/lib/remotion/cli.ts +34 -0
- package/package.json +20 -3
- package/pipeline/cookbooks/scripts/animate-frames-parallel.ts +83 -0
- package/pipeline/cookbooks/scripts/combine-scenes.sh +53 -0
- package/pipeline/cookbooks/scripts/generate-frames-parallel.ts +98 -0
- package/pipeline/cookbooks/scripts/still-to-video.sh +37 -0
- package/pipeline/cookbooks/text-to-tiktok.md +669 -0
- package/scripts/.gitkeep +0 -0
- package/service/music/index.ts +29 -14
- package/tsconfig.json +1 -1
- package/HIGGSFIELD_REWRITE_SUMMARY.md +0 -300
- package/TEST_RESULTS.md +0 -122
- package/output.txt +0 -1
- package/scripts/produce-menopause-campaign.sh +0 -202
- package/test-import.ts +0 -7
- package/test-services.ts +0 -97
package/lib/remotion/SKILL.md
CHANGED
|
@@ -301,6 +301,30 @@ bun run lib/remotion/index.ts render <root-file.tsx> <comp-id> <output.mp4>
|
|
|
301
301
|
bun run lib/remotion/index.ts still <root-file.tsx> <comp-id> <frame> <out.png>
|
|
302
302
|
```
|
|
303
303
|
|
|
304
|
+
### remotion studio (visual preview & debugging)
|
|
305
|
+
```bash
|
|
306
|
+
# launch studio to preview compositions visually
|
|
307
|
+
bun remotion studio lib/remotion/compositions/MyVideo.root.tsx --public-dir=lib/remotion/public
|
|
308
|
+
|
|
309
|
+
# studio features:
|
|
310
|
+
# - scrub through timeline
|
|
311
|
+
# - see all sequences visually with names
|
|
312
|
+
# - debug timing issues
|
|
313
|
+
# - preview renders before committing
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### direct render with bun (alternative)
|
|
317
|
+
```bash
|
|
318
|
+
# render full video
|
|
319
|
+
bun remotion render lib/remotion/compositions/MyVideo.root.tsx MyVideo output.mp4 --public-dir=lib/remotion/public
|
|
320
|
+
|
|
321
|
+
# render at lower resolution for preview (faster)
|
|
322
|
+
bun remotion render lib/remotion/compositions/MyVideo.root.tsx MyVideo preview.mp4 --public-dir=lib/remotion/public --scale=0.5
|
|
323
|
+
|
|
324
|
+
# render specific frame range for testing
|
|
325
|
+
bun remotion render lib/remotion/compositions/MyVideo.root.tsx MyVideo test.mp4 --public-dir=lib/remotion/public --frames=0-60
|
|
326
|
+
```
|
|
327
|
+
|
|
304
328
|
### lib/ffmpeg.ts
|
|
305
329
|
```bash
|
|
306
330
|
# get video metadata
|
|
@@ -575,28 +599,130 @@ return (
|
|
|
575
599
|
);
|
|
576
600
|
```
|
|
577
601
|
|
|
578
|
-
###
|
|
602
|
+
### looping videos (when scene > video duration)
|
|
603
|
+
when scene duration exceeds video file duration, video freezes on last frame. use `<Loop>`:
|
|
604
|
+
|
|
579
605
|
```typescript
|
|
580
|
-
|
|
581
|
-
const subtitles = parseSRT(srtContent);
|
|
606
|
+
import { Loop, OffthreadVideo } from "remotion";
|
|
582
607
|
|
|
583
|
-
//
|
|
584
|
-
const
|
|
585
|
-
|
|
586
|
-
|
|
608
|
+
// define actual video durations (from ffprobe)
|
|
609
|
+
const VIDEO_DURATIONS_SECONDS: Record<number, number> = {
|
|
610
|
+
1: 5.041667,
|
|
611
|
+
2: 10.041667,
|
|
612
|
+
// ...
|
|
613
|
+
};
|
|
587
614
|
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
615
|
+
// looping video component
|
|
616
|
+
const LoopingVideo: React.FC<{
|
|
617
|
+
src: string;
|
|
618
|
+
loopDurationInFrames: number;
|
|
619
|
+
}> = ({ src, loopDurationInFrames }) => {
|
|
620
|
+
return (
|
|
621
|
+
<Loop durationInFrames={loopDurationInFrames}>
|
|
622
|
+
<OffthreadVideo src={src} muted />
|
|
623
|
+
</Loop>
|
|
624
|
+
);
|
|
625
|
+
};
|
|
591
626
|
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
627
|
+
// usage in render
|
|
628
|
+
const loopDuration = VIDEO_DURATIONS_SECONDS[scene.video];
|
|
629
|
+
const loopDurationInFrames = Math.round(loopDuration * fps);
|
|
630
|
+
|
|
631
|
+
<Sequence from={startFrame} durationInFrames={sceneDurationFrames}>
|
|
632
|
+
<LoopingVideo
|
|
633
|
+
src={staticFile(`scene${scene.video}_video.mp4`)}
|
|
634
|
+
loopDurationInFrames={loopDurationInFrames}
|
|
635
|
+
/>
|
|
636
|
+
</Sequence>
|
|
637
|
+
```
|
|
638
|
+
|
|
639
|
+
**critical**: `<Loop durationInFrames>` takes the VIDEO's duration (how long before it loops), not the scene's duration. the Sequence controls how long the scene plays.
|
|
640
|
+
|
|
641
|
+
### video with word-by-word captions (tiktok style)
|
|
642
|
+
```typescript
|
|
643
|
+
// parse SRT content to captions array
|
|
644
|
+
interface Caption {
|
|
645
|
+
id: number;
|
|
646
|
+
startMs: number;
|
|
647
|
+
endMs: number;
|
|
648
|
+
text: string;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
const parseTimestamp = (ts: string): number => {
|
|
652
|
+
const [time, ms] = ts.split(",");
|
|
653
|
+
const [h, m, s] = (time ?? "0:0:0").split(":").map(Number);
|
|
654
|
+
return (h ?? 0) * 3600000 + (m ?? 0) * 60000 + (s ?? 0) * 1000 + Number(ms ?? 0);
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
const parseSRT = (content: string): Caption[] => {
|
|
658
|
+
const blocks = content.trim().split(/\n\n+/);
|
|
659
|
+
return blocks.map((block) => {
|
|
660
|
+
const lines = block.split("\n");
|
|
661
|
+
const id = Number(lines[0]);
|
|
662
|
+
const timeLine = lines[1] ?? "00:00:00,000 --> 00:00:00,000";
|
|
663
|
+
const [startTs, endTs] = timeLine.split(" --> ");
|
|
664
|
+
const text = lines.slice(2).join(" ");
|
|
665
|
+
return {
|
|
666
|
+
id,
|
|
667
|
+
startMs: parseTimestamp(startTs ?? "00:00:00,000"),
|
|
668
|
+
endMs: parseTimestamp(endTs ?? "00:00:00,000"),
|
|
669
|
+
text,
|
|
670
|
+
};
|
|
671
|
+
});
|
|
672
|
+
};
|
|
673
|
+
|
|
674
|
+
// group words into chunks for display (4 words at a time)
|
|
675
|
+
const WORDS_PER_CHUNK = 4;
|
|
676
|
+
const groupCaptions = (captions: Caption[]) => {
|
|
677
|
+
const groups: { words: Caption[]; startMs: number; endMs: number }[] = [];
|
|
678
|
+
for (let i = 0; i < captions.length; i += WORDS_PER_CHUNK) {
|
|
679
|
+
const chunk = captions.slice(i, i + WORDS_PER_CHUNK);
|
|
680
|
+
const first = chunk[0];
|
|
681
|
+
const last = chunk[chunk.length - 1];
|
|
682
|
+
if (first && last) {
|
|
683
|
+
groups.push({ words: chunk, startMs: first.startMs, endMs: last.endMs });
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
return groups;
|
|
687
|
+
};
|
|
688
|
+
|
|
689
|
+
// TikTok-style captions component with word highlighting
|
|
690
|
+
const Captions: React.FC = () => {
|
|
691
|
+
const frame = useCurrentFrame();
|
|
692
|
+
const { fps } = useVideoConfig();
|
|
693
|
+
const currentTimeMs = (frame / fps) * 1000;
|
|
694
|
+
|
|
695
|
+
const currentGroup = CAPTION_GROUPS.find(
|
|
696
|
+
(group) => currentTimeMs >= group.startMs && currentTimeMs <= group.endMs
|
|
697
|
+
);
|
|
698
|
+
|
|
699
|
+
if (!currentGroup) return null;
|
|
700
|
+
|
|
701
|
+
return (
|
|
702
|
+
<div style={{ position: "absolute", bottom: "15%", left: "50%", transform: "translateX(-50%)", width: "90%", textAlign: "center" }}>
|
|
703
|
+
<div style={{ display: "flex", flexWrap: "wrap", justifyContent: "center", gap: "24px" }}>
|
|
704
|
+
{currentGroup.words.map((word) => {
|
|
705
|
+
const isActive = currentTimeMs >= word.startMs && currentTimeMs <= word.endMs;
|
|
706
|
+
const isPast = currentTimeMs > word.endMs;
|
|
707
|
+
return (
|
|
708
|
+
<span key={word.id} style={{
|
|
709
|
+
fontFamily: "Arial Black, sans-serif",
|
|
710
|
+
fontSize: "64px",
|
|
711
|
+
fontWeight: 900,
|
|
712
|
+
letterSpacing: "0.05em",
|
|
713
|
+
color: isActive ? "#FFD700" : isPast ? "#FFFFFF" : "#AAAAAA",
|
|
714
|
+
textShadow: isActive ? "0 0 20px #FFD700, 0 4px 8px rgba(0,0,0,0.8)" : "0 4px 8px rgba(0,0,0,0.8)",
|
|
715
|
+
transform: isActive ? "scale(1.1)" : "scale(1)",
|
|
716
|
+
textTransform: "uppercase",
|
|
717
|
+
}}>
|
|
718
|
+
{word.text}
|
|
719
|
+
</span>
|
|
720
|
+
);
|
|
721
|
+
})}
|
|
722
|
+
</div>
|
|
723
|
+
</div>
|
|
724
|
+
);
|
|
725
|
+
};
|
|
600
726
|
```
|
|
601
727
|
|
|
602
728
|
### sequential video concatenation
|
|
@@ -618,6 +744,72 @@ return (
|
|
|
618
744
|
);
|
|
619
745
|
```
|
|
620
746
|
|
|
747
|
+
### scene-based timeline with named sequences
|
|
748
|
+
```typescript
|
|
749
|
+
// define scenes with timing (separate id from video file for reuse)
|
|
750
|
+
const SCENES = [
|
|
751
|
+
// Scene 1: Title/Hook - protagonist sitting alone on bench at night
|
|
752
|
+
{ id: 1, video: 1, name: "1. Title/Hook", start: 0, duration: 3.5 },
|
|
753
|
+
|
|
754
|
+
// Scene 2: Young Love - protagonist and first girl walking in park
|
|
755
|
+
{ id: 2, video: 2, name: "2. Young Love", start: 3.5, duration: 6.5 },
|
|
756
|
+
|
|
757
|
+
// Scene 3: Flashback - reuse scene 5 video with B&W effect
|
|
758
|
+
{ id: 3, video: 5, name: "3. Flashback", start: 54.5, duration: 6.5, flashback: true },
|
|
759
|
+
|
|
760
|
+
// Scene 4: Same video used again (video 13 appears twice)
|
|
761
|
+
{ id: 4, video: 13, name: "4. Years Pass", start: 120, duration: 15 },
|
|
762
|
+
];
|
|
763
|
+
|
|
764
|
+
// render with names visible in Remotion Studio
|
|
765
|
+
{SCENES.map((scene) => {
|
|
766
|
+
const startFrame = Math.round(scene.start * fps);
|
|
767
|
+
const durationFrames = Math.round(scene.duration * fps);
|
|
768
|
+
|
|
769
|
+
return (
|
|
770
|
+
<Sequence
|
|
771
|
+
key={scene.id}
|
|
772
|
+
name={scene.name} // <-- shows in Studio timeline!
|
|
773
|
+
from={startFrame}
|
|
774
|
+
durationInFrames={durationFrames}
|
|
775
|
+
>
|
|
776
|
+
<LoopingVideo
|
|
777
|
+
src={staticFile(`scene${scene.video}_video.mp4`)}
|
|
778
|
+
flashback={scene.flashback}
|
|
779
|
+
/>
|
|
780
|
+
</Sequence>
|
|
781
|
+
);
|
|
782
|
+
})}
|
|
783
|
+
```
|
|
784
|
+
|
|
785
|
+
### flashback effect (B&W jittery style)
|
|
786
|
+
```typescript
|
|
787
|
+
// video component with optional flashback effect
|
|
788
|
+
const LoopingVideo: React.FC<{ src: string; flashback?: boolean }> = ({ src, flashback }) => {
|
|
789
|
+
const frame = useCurrentFrame();
|
|
790
|
+
|
|
791
|
+
// Flashback effect: B&W with jittery/glitchy feel
|
|
792
|
+
const flashbackStyle = flashback ? {
|
|
793
|
+
filter: "grayscale(100%) contrast(1.2) brightness(0.9)",
|
|
794
|
+
// Subtle jitter using frame-based transform
|
|
795
|
+
transform: `translate(${Math.sin(frame * 0.5) * 2}px, ${Math.cos(frame * 0.7) * 1.5}px)`,
|
|
796
|
+
} : {};
|
|
797
|
+
|
|
798
|
+
return (
|
|
799
|
+
<OffthreadVideo
|
|
800
|
+
src={src}
|
|
801
|
+
style={{
|
|
802
|
+
width: "100%",
|
|
803
|
+
height: "100%",
|
|
804
|
+
objectFit: "cover",
|
|
805
|
+
...flashbackStyle,
|
|
806
|
+
}}
|
|
807
|
+
muted
|
|
808
|
+
/>
|
|
809
|
+
);
|
|
810
|
+
};
|
|
811
|
+
```
|
|
812
|
+
|
|
621
813
|
### crossfade transition between videos
|
|
622
814
|
```typescript
|
|
623
815
|
const transitionStart = 140;
|
|
@@ -719,16 +911,38 @@ const captionOpacity = interpolate(
|
|
|
719
911
|
|
|
720
912
|
### deprecated components warnings
|
|
721
913
|
- **warning**: `Video` and `Audio` are deprecated
|
|
722
|
-
- **fix**: use `OffthreadVideo` instead of `Video`, `
|
|
914
|
+
- **fix**: use `OffthreadVideo` instead of `Video`, use `Html5Audio` instead of `Audio`
|
|
723
915
|
- **example**:
|
|
724
916
|
```tsx
|
|
725
917
|
// ❌ deprecated
|
|
726
|
-
import { Video } from "remotion";
|
|
918
|
+
import { Video, Audio } from "remotion";
|
|
727
919
|
<Video src={staticFile("video.mp4")} />
|
|
920
|
+
<Audio src={staticFile("audio.mp3")} />
|
|
728
921
|
|
|
729
922
|
// ✅ recommended
|
|
730
|
-
import { OffthreadVideo } from "remotion";
|
|
923
|
+
import { OffthreadVideo, Html5Audio } from "remotion";
|
|
731
924
|
<OffthreadVideo src={staticFile("video.mp4")} />
|
|
925
|
+
<Html5Audio src={staticFile("audio.mp3")} />
|
|
926
|
+
```
|
|
927
|
+
|
|
928
|
+
### video freezes on last frame
|
|
929
|
+
- **error**: when scene duration > video file duration, video freezes on last frame
|
|
930
|
+
- **cause**: `OffthreadVideo` naturally stops at end of video
|
|
931
|
+
- **fix**: wrap in `<Loop>` with the VIDEO's duration (not scene duration)
|
|
932
|
+
- **example**:
|
|
933
|
+
```tsx
|
|
934
|
+
import { Loop, OffthreadVideo } from "remotion";
|
|
935
|
+
|
|
936
|
+
// ❌ wrong - scene duration
|
|
937
|
+
<Loop durationInFrames={sceneDurationFrames}>
|
|
938
|
+
<OffthreadVideo src={video} />
|
|
939
|
+
</Loop>
|
|
940
|
+
|
|
941
|
+
// ✅ correct - video file's actual duration
|
|
942
|
+
const videoDurationFrames = Math.round(5.04 * fps); // from ffprobe
|
|
943
|
+
<Loop durationInFrames={videoDurationFrames}>
|
|
944
|
+
<OffthreadVideo src={video} />
|
|
945
|
+
</Loop>
|
|
732
946
|
```
|
|
733
947
|
|
|
734
948
|
### type errors with array indexing
|
|
@@ -821,3 +1035,8 @@ const captionOpacity = interpolate(
|
|
|
821
1035
|
11. **handle fps differences** - adjust startFrom when concatenating videos with different fps
|
|
822
1036
|
12. **use descriptive ids** - make composition names clear and unique
|
|
823
1037
|
13. **batch render with props** - for multiple variations, register multiple compositions with unique defaultProps instead of file overwriting
|
|
1038
|
+
14. **name your sequences** - add `name` prop to `<Sequence>` components for visibility in Studio timeline
|
|
1039
|
+
15. **separate video id from scene id** - allows reusing same video in multiple scenes (e.g., flashbacks)
|
|
1040
|
+
16. **use studio for debugging** - `bun remotion studio` lets you scrub through timeline and see named sequences
|
|
1041
|
+
17. **add scene comments** - document what each scene contains in code for easy reference
|
|
1042
|
+
18. **embed SRT content directly** - avoids file loading issues in compositions (parse once, use everywhere)
|
package/lib/remotion/cli.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
1
2
|
import {
|
|
2
3
|
createComposition,
|
|
3
4
|
getCompositionsList,
|
|
@@ -17,6 +18,7 @@ usage:
|
|
|
17
18
|
|
|
18
19
|
commands:
|
|
19
20
|
create <name> setup composition directory
|
|
21
|
+
studio <root-file.tsx> open remotion studio
|
|
20
22
|
compositions <root-file.tsx> list all compositions
|
|
21
23
|
render <root-file.tsx> <comp-id> <output.mp4> render video
|
|
22
24
|
still <root-file.tsx> <comp-id> <frame> <out.png> render still frame
|
|
@@ -24,6 +26,7 @@ commands:
|
|
|
24
26
|
|
|
25
27
|
examples:
|
|
26
28
|
bun run lib/remotion/index.ts create MyVideo
|
|
29
|
+
bun run lib/remotion/index.ts studio lib/remotion/compositions/MyVideo.root.tsx
|
|
27
30
|
bun run lib/remotion/index.ts compositions lib/remotion/compositions/MyVideo.root.tsx
|
|
28
31
|
bun run lib/remotion/index.ts render lib/remotion/compositions/MyVideo.root.tsx Demo output.mp4
|
|
29
32
|
bun run lib/remotion/index.ts still lib/remotion/compositions/MyVideo.root.tsx Demo 30 frame.png
|
|
@@ -49,6 +52,37 @@ requirements:
|
|
|
49
52
|
break;
|
|
50
53
|
}
|
|
51
54
|
|
|
55
|
+
case "studio": {
|
|
56
|
+
const entryPoint = args[1];
|
|
57
|
+
|
|
58
|
+
if (!entryPoint) {
|
|
59
|
+
throw new Error("entry point is required");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const publicDir = join(process.cwd(), "lib/remotion/public");
|
|
63
|
+
const { spawn } = await import("node:child_process");
|
|
64
|
+
|
|
65
|
+
console.log(`[remotion] starting studio with public dir: ${publicDir}`);
|
|
66
|
+
|
|
67
|
+
const studio = spawn(
|
|
68
|
+
"bun",
|
|
69
|
+
["remotion", "studio", entryPoint, "--public-dir", publicDir],
|
|
70
|
+
{
|
|
71
|
+
stdio: "inherit",
|
|
72
|
+
cwd: process.cwd(),
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
studio.on("error", (err) => {
|
|
77
|
+
console.error("[remotion] studio error:", err);
|
|
78
|
+
process.exit(1);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// keep process alive
|
|
82
|
+
await new Promise(() => {});
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
|
|
52
86
|
case "compositions": {
|
|
53
87
|
const entryPoint = args[1];
|
|
54
88
|
|
package/package.json
CHANGED
|
@@ -7,11 +7,27 @@
|
|
|
7
7
|
},
|
|
8
8
|
"scripts": {
|
|
9
9
|
"lint": "biome check .",
|
|
10
|
-
"format": "biome format --write ."
|
|
10
|
+
"format": "biome format --write .",
|
|
11
|
+
"type-check": "tsc --noEmit",
|
|
12
|
+
"prepare": "husky install",
|
|
13
|
+
"size": "size-limit"
|
|
14
|
+
},
|
|
15
|
+
"lint-staged": {
|
|
16
|
+
"*.{js,ts,tsx}": [
|
|
17
|
+
"biome check --write --no-errors-on-unmatched"
|
|
18
|
+
],
|
|
19
|
+
"*.{json,md}": [
|
|
20
|
+
"biome format --write"
|
|
21
|
+
]
|
|
11
22
|
},
|
|
12
23
|
"devDependencies": {
|
|
13
24
|
"@biomejs/biome": "^2.3.7",
|
|
14
|
-
"@
|
|
25
|
+
"@commitlint/cli": "^20.1.0",
|
|
26
|
+
"@commitlint/config-conventional": "^20.0.0",
|
|
27
|
+
"@size-limit/preset-small-lib": "^11.2.0",
|
|
28
|
+
"@types/bun": "latest",
|
|
29
|
+
"husky": "^9.1.7",
|
|
30
|
+
"lint-staged": "^16.2.7"
|
|
15
31
|
},
|
|
16
32
|
"peerDependencies": {
|
|
17
33
|
"typescript": "^5"
|
|
@@ -26,6 +42,7 @@
|
|
|
26
42
|
"@higgsfield/client": "^0.1.2",
|
|
27
43
|
"@remotion/cli": "^4.0.377",
|
|
28
44
|
"@types/fluent-ffmpeg": "^2.1.28",
|
|
45
|
+
"@vargai/sdk": "^0.1.1",
|
|
29
46
|
"ai": "^5.0.98",
|
|
30
47
|
"citty": "^0.1.6",
|
|
31
48
|
"fluent-ffmpeg": "^2.1.3",
|
|
@@ -35,7 +52,7 @@
|
|
|
35
52
|
"remotion": "^4.0.377",
|
|
36
53
|
"replicate": "^1.4.0"
|
|
37
54
|
},
|
|
38
|
-
"version": "0.1.
|
|
55
|
+
"version": "0.1.2",
|
|
39
56
|
"exports": {
|
|
40
57
|
".": "./index.ts"
|
|
41
58
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Animate multiple frames in parallel using kling
|
|
3
|
+
* Usage: bun run pipeline/cookbooks/scripts/animate-frames-parallel.ts
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { fal } from "@fal-ai/client";
|
|
7
|
+
|
|
8
|
+
interface VideoConfig {
|
|
9
|
+
name: string;
|
|
10
|
+
framePath: string;
|
|
11
|
+
prompt: string;
|
|
12
|
+
duration?: "5" | "10";
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async function animateFrames(configs: VideoConfig[], outputDir: string) {
|
|
16
|
+
console.log(`Animating ${configs.length} frames in parallel...\n`);
|
|
17
|
+
|
|
18
|
+
// Upload all frames first
|
|
19
|
+
const frameUrls: string[] = [];
|
|
20
|
+
for (const config of configs) {
|
|
21
|
+
const url = await fal.storage.upload(Bun.file(config.framePath));
|
|
22
|
+
frameUrls.push(url);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const promises = configs.map((config, i) => {
|
|
26
|
+
return fal.subscribe("fal-ai/kling-video/v2.5-turbo/pro/image-to-video", {
|
|
27
|
+
input: {
|
|
28
|
+
prompt: config.prompt + ", NO talking NO lip movement",
|
|
29
|
+
image_url: frameUrls[i]!,
|
|
30
|
+
duration: config.duration || "5",
|
|
31
|
+
// note: aspect_ratio is determined by input image dimensions
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const results = await Promise.all(promises);
|
|
37
|
+
|
|
38
|
+
for (let i = 0; i < results.length; i++) {
|
|
39
|
+
const result = results[i] as { data?: { video?: { url?: string } } };
|
|
40
|
+
const url = result.data?.video?.url;
|
|
41
|
+
if (url) {
|
|
42
|
+
const response = await fetch(url);
|
|
43
|
+
const buffer = await response.arrayBuffer();
|
|
44
|
+
await Bun.write(`${outputDir}/${configs[i]!.name}_video.mp4`, buffer);
|
|
45
|
+
console.log(`${configs[i]!.name}_video.mp4 saved`);
|
|
46
|
+
} else {
|
|
47
|
+
console.error(`No URL for ${configs[i]!.name}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
console.log("\nAll videos saved!");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Example usage:
|
|
55
|
+
async function main() {
|
|
56
|
+
const outputDir = "media/girl-ruined-you";
|
|
57
|
+
|
|
58
|
+
const configs: VideoConfig[] = [
|
|
59
|
+
{
|
|
60
|
+
name: "scene6",
|
|
61
|
+
framePath: `${outputDir}/scene6_frame.jpg`,
|
|
62
|
+
prompt:
|
|
63
|
+
"3D pixar animation, two cats meet eyes in coffee shop, warm romantic moment",
|
|
64
|
+
duration: "5",
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
name: "scene7",
|
|
68
|
+
framePath: `${outputDir}/scene7_frame.jpg`,
|
|
69
|
+
prompt: "3D pixar animation, two cats walking together, sunset, romantic",
|
|
70
|
+
duration: "5",
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
name: "scene14",
|
|
74
|
+
framePath: `${outputDir}/scene14_frame.jpg`,
|
|
75
|
+
prompt: "3D pixar animation, cat looks at sunrise, hopeful realization",
|
|
76
|
+
duration: "5",
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
await animateFrames(configs, outputDir);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Combine multiple scene videos with audio clips
|
|
3
|
+
# Usage: ./combine-scenes.sh <project_dir>
|
|
4
|
+
|
|
5
|
+
PROJECT_DIR=${1:-"media/girl-ruined-you"}
|
|
6
|
+
|
|
7
|
+
# Scene timing configuration (adjust as needed)
|
|
8
|
+
# Format: scene_num:start_time:duration
|
|
9
|
+
SCENES=(
|
|
10
|
+
"1:0:3.5"
|
|
11
|
+
"2:3.5:6.5"
|
|
12
|
+
"3:10:10"
|
|
13
|
+
"4:20:15"
|
|
14
|
+
"5:35:7"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
echo "Extracting audio clips..."
|
|
18
|
+
for scene_config in "${SCENES[@]}"; do
|
|
19
|
+
IFS=':' read -r num start dur <<< "$scene_config"
|
|
20
|
+
ffmpeg -y -i "$PROJECT_DIR/voiceover.mp3" -ss "$start" -t "$dur" "$PROJECT_DIR/audio_scene${num}.mp3" 2>/dev/null
|
|
21
|
+
echo " audio_scene${num}.mp3 ($dur sec)"
|
|
22
|
+
done
|
|
23
|
+
|
|
24
|
+
echo ""
|
|
25
|
+
echo "Combining videos with audio..."
|
|
26
|
+
for scene_config in "${SCENES[@]}"; do
|
|
27
|
+
IFS=':' read -r num start dur <<< "$scene_config"
|
|
28
|
+
|
|
29
|
+
# Calculate loop count needed (5s videos)
|
|
30
|
+
loops=$(echo "($dur / 5) - 1" | bc)
|
|
31
|
+
if [ "$loops" -lt 0 ]; then loops=0; fi
|
|
32
|
+
|
|
33
|
+
ffmpeg -y -stream_loop "$loops" -i "$PROJECT_DIR/scene${num}_video.mp4" \
|
|
34
|
+
-i "$PROJECT_DIR/audio_scene${num}.mp3" \
|
|
35
|
+
-t "$dur" -c:v libx264 -preset fast -crf 20 -c:a aac -b:a 128k -shortest \
|
|
36
|
+
"$PROJECT_DIR/scene${num}_final.mp4" 2>/dev/null
|
|
37
|
+
echo " scene${num}_final.mp4"
|
|
38
|
+
done
|
|
39
|
+
|
|
40
|
+
echo ""
|
|
41
|
+
echo "Creating concat file..."
|
|
42
|
+
rm -f "$PROJECT_DIR/scenes.txt"
|
|
43
|
+
for scene_config in "${SCENES[@]}"; do
|
|
44
|
+
IFS=':' read -r num start dur <<< "$scene_config"
|
|
45
|
+
echo "file 'scene${num}_final.mp4'" >> "$PROJECT_DIR/scenes.txt"
|
|
46
|
+
done
|
|
47
|
+
|
|
48
|
+
echo "Concatenating all scenes..."
|
|
49
|
+
cd "$PROJECT_DIR" && ffmpeg -y -f concat -safe 0 -i scenes.txt -c copy combined_scenes.mp4 2>/dev/null
|
|
50
|
+
|
|
51
|
+
duration=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 combined_scenes.mp4)
|
|
52
|
+
echo ""
|
|
53
|
+
echo "Done! combined_scenes.mp4 ($duration sec)"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generate multiple scene frames in parallel using flux kontext
|
|
3
|
+
* Usage: bun run pipeline/cookbooks/scripts/generate-frames-parallel.ts
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { fal } from "@fal-ai/client";
|
|
7
|
+
|
|
8
|
+
interface FrameConfig {
|
|
9
|
+
name: string;
|
|
10
|
+
prompt: string;
|
|
11
|
+
imageUrls: string[]; // character reference URLs
|
|
12
|
+
multi?: boolean; // use kontext/multi for multiple characters
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async function generateFrames(configs: FrameConfig[], outputDir: string) {
|
|
16
|
+
console.log(`Generating ${configs.length} frames in parallel...\n`);
|
|
17
|
+
|
|
18
|
+
const promises = configs.map((config) => {
|
|
19
|
+
if (config.multi) {
|
|
20
|
+
return fal.subscribe("fal-ai/flux-pro/kontext/multi", {
|
|
21
|
+
input: {
|
|
22
|
+
prompt: config.prompt,
|
|
23
|
+
image_urls: config.imageUrls,
|
|
24
|
+
aspect_ratio: "9:16" as const,
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
} else {
|
|
28
|
+
return fal.subscribe("fal-ai/flux-pro/kontext", {
|
|
29
|
+
input: {
|
|
30
|
+
prompt: config.prompt,
|
|
31
|
+
image_url: config.imageUrls[0]!,
|
|
32
|
+
aspect_ratio: "9:16" as const,
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const results = await Promise.all(promises);
|
|
39
|
+
|
|
40
|
+
for (let i = 0; i < results.length; i++) {
|
|
41
|
+
const result = results[i] as {
|
|
42
|
+
data?: { images?: Array<{ url?: string }> };
|
|
43
|
+
};
|
|
44
|
+
const url = result.data?.images?.[0]?.url;
|
|
45
|
+
if (url) {
|
|
46
|
+
const response = await fetch(url);
|
|
47
|
+
const buffer = await response.arrayBuffer();
|
|
48
|
+
await Bun.write(`${outputDir}/${configs[i]!.name}_frame.jpg`, buffer);
|
|
49
|
+
console.log(`${configs[i]!.name}_frame.jpg saved`);
|
|
50
|
+
} else {
|
|
51
|
+
console.error(`No URL for ${configs[i]!.name}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
console.log("\nAll frames saved!");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Example usage:
|
|
59
|
+
async function main() {
|
|
60
|
+
const outputDir = "media/girl-ruined-you";
|
|
61
|
+
|
|
62
|
+
// Upload character references first
|
|
63
|
+
const protagonist = await fal.storage.upload(
|
|
64
|
+
Bun.file(`${outputDir}/cat_protagonist.png`),
|
|
65
|
+
);
|
|
66
|
+
const secondGirl = await fal.storage.upload(
|
|
67
|
+
Bun.file(`${outputDir}/cat_second_girl.png`),
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
const configs: FrameConfig[] = [
|
|
71
|
+
{
|
|
72
|
+
name: "scene6",
|
|
73
|
+
prompt:
|
|
74
|
+
"3D pixar style: male cat in hoodie (first) and elegant female cat (second) meeting eyes in coffee shop, warm golden lighting, vertical portrait 9:16",
|
|
75
|
+
imageUrls: [protagonist, secondGirl],
|
|
76
|
+
multi: true,
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
name: "scene7",
|
|
80
|
+
prompt:
|
|
81
|
+
"3D pixar style: male cat and female cat walking together, sunset, romantic, vertical portrait 9:16",
|
|
82
|
+
imageUrls: [protagonist, secondGirl],
|
|
83
|
+
multi: true,
|
|
84
|
+
},
|
|
85
|
+
// Single character scene
|
|
86
|
+
{
|
|
87
|
+
name: "scene14",
|
|
88
|
+
prompt:
|
|
89
|
+
"Place this cat looking at sunrise through window, hopeful, vertical portrait 9:16",
|
|
90
|
+
imageUrls: [protagonist],
|
|
91
|
+
multi: false,
|
|
92
|
+
},
|
|
93
|
+
];
|
|
94
|
+
|
|
95
|
+
await generateFrames(configs, outputDir);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Convert still frame to video with ken burns effect (slow zoom)
|
|
3
|
+
# Usage: ./still-to-video.sh <input.jpg> <output.mp4> <duration> [zoom_direction]
|
|
4
|
+
# zoom_direction: in (default), out
|
|
5
|
+
|
|
6
|
+
INPUT=$1
|
|
7
|
+
OUTPUT=$2
|
|
8
|
+
DURATION=$3
|
|
9
|
+
ZOOM=${4:-"in"}
|
|
10
|
+
|
|
11
|
+
if [ -z "$INPUT" ] || [ -z "$OUTPUT" ] || [ -z "$DURATION" ]; then
|
|
12
|
+
echo "Usage: ./still-to-video.sh <input.jpg> <output.mp4> <duration> [in|out]"
|
|
13
|
+
exit 1
|
|
14
|
+
fi
|
|
15
|
+
|
|
16
|
+
# Get input dimensions
|
|
17
|
+
WIDTH=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of csv=p=0 "$INPUT")
|
|
18
|
+
HEIGHT=$(ffprobe -v error -select_streams v:0 -show_entries stream=height -of csv=p=0 "$INPUT")
|
|
19
|
+
|
|
20
|
+
echo "Creating $DURATION sec video from $INPUT ($WIDTH x $HEIGHT)..."
|
|
21
|
+
|
|
22
|
+
if [ "$ZOOM" = "out" ]; then
|
|
23
|
+
# Zoom out: start zoomed in, end at normal
|
|
24
|
+
FILTER="zoompan=z='1.2-0.2*on/(${DURATION}*25)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${DURATION}*25:s=${WIDTH}x${HEIGHT}:fps=25"
|
|
25
|
+
else
|
|
26
|
+
# Zoom in: start normal, end zoomed
|
|
27
|
+
FILTER="zoompan=z='1+0.2*on/(${DURATION}*25)':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=${DURATION}*25:s=${WIDTH}x${HEIGHT}:fps=25"
|
|
28
|
+
fi
|
|
29
|
+
|
|
30
|
+
ffmpeg -y -loop 1 -i "$INPUT" \
|
|
31
|
+
-vf "$FILTER" \
|
|
32
|
+
-t "$DURATION" \
|
|
33
|
+
-c:v libx264 -preset fast -crf 20 \
|
|
34
|
+
-pix_fmt yuv420p \
|
|
35
|
+
"$OUTPUT"
|
|
36
|
+
|
|
37
|
+
echo "Done: $OUTPUT"
|