@autoaiai/aicut 0.0.0-stage → 0.1.11

package/skills/remotion-resemble-ai/rules/resemble-captions.md

@@ -0,0 +1,350 @@
+---
+name: resemble-captions
+description: Using Resemble.ai timestamps to create synchronized captions
+metadata:
+  tags: resemble, captions, subtitles, timestamps, synchronization, tiktok
+---
+
+# Creating Synchronized Captions
+
+> **⚠️ OPT-IN FEATURE:** Captions are disabled by default. Only implement captions when the user explicitly requests them (e.g., "add captions", "include subtitles", "word highlighting").
+
+Resemble.ai provides character-level timestamps with synthesized audio, enabling precise caption synchronization.
+
+## Understanding Timestamp Data
+
+The synchronous API returns `audio_timestamps`:
+
+```typescript
+interface AudioTimestamps {
+  graph_chars: string[];      // Individual characters
+  graph_times: [number, number][]; // [start, end] times in seconds
+  phon_chars: string[];       // Phonemes (legacy)
+  phon_times: [number, number][];
+}
+```
+
+Example response:
+
+```json
+{
+  "graph_chars": ["H", "e", "l", "l", "o"],
+  "graph_times": [[0.0, 0.12], [0.12, 0.24], [0.24, 0.36], [0.36, 0.48], [0.48, 0.6]]
+}
+```
+
+## Converting Timestamps to Words
+
+```typescript
+// src/resemble/captions.ts
+interface Word {
+  text: string;
+  startTime: number;
+  endTime: number;
+}
+
+interface AudioTimestamps {
+  graph_chars: string[];
+  graph_times: [number, number][];
+}
+
+export function timestampsToWords(timestamps: AudioTimestamps): Word[] {
+  const words: Word[] = [];
+  let currentWord = '';
+  let wordStart = 0;
+  let wordEnd = 0;
+
+  for (let i = 0; i < timestamps.graph_chars.length; i++) {
+    const char = timestamps.graph_chars[i];
+    const [start, end] = timestamps.graph_times[i];
+
+    if (char === ' ' || char === '\n') {
+      // End of word
+      if (currentWord.trim()) {
+        words.push({
+          text: currentWord.trim(),
+          startTime: wordStart,
+          endTime: wordEnd,
+        });
+      }
+      currentWord = '';
+      wordStart = end;
+    } else {
+      if (!currentWord) {
+        wordStart = start;
+      }
+      currentWord += char;
+      wordEnd = end;
+    }
+  }
+
+  // Don't forget the last word
+  if (currentWord.trim()) {
+    words.push({
+      text: currentWord.trim(),
+      startTime: wordStart,
+      endTime: wordEnd,
+    });
+  }
+
+  return words;
+}
+```
+
+## Caption Component for Remotion
+
+```tsx
+// src/components/Captions.tsx
+import { useCurrentFrame, useVideoConfig } from 'remotion';
+
+interface Word {
+  text: string;
+  startTime: number;
+  endTime: number;
+}
+
+interface CaptionsProps {
+  words: Word[];
+  style?: React.CSSProperties;
+}
+
+export const Captions: React.FC<CaptionsProps> = ({ words, style }) => {
+  const frame = useCurrentFrame();
+  const { fps } = useVideoConfig();
+
+  const currentTime = frame / fps;
+
+  // Find current word
+  const currentWordIndex = words.findIndex(
+    (word) => currentTime >= word.startTime && currentTime <= word.endTime
+  );
+
+  // Get words to display (show a few words at a time)
+  const windowSize = 5;
+  const startIndex = Math.max(0, currentWordIndex - 2);
+  const visibleWords = words.slice(startIndex, startIndex + windowSize);
+
+  return (
+    <div
+      style={{
+        position: 'absolute',
+        bottom: 100,
+        left: 0,
+        right: 0,
+        textAlign: 'center',
+        padding: '0 40px',
+        ...style,
+      }}
+    >
+      <div
+        style={{
+          display: 'inline-flex',
+          flexWrap: 'wrap',
+          justifyContent: 'center',
+          gap: '8px',
+        }}
+      >
+        {visibleWords.map((word, idx) => {
+          const absoluteIndex = startIndex + idx;
+          const isActive = absoluteIndex === currentWordIndex;
+          const isPast = currentTime > word.endTime;
+
+          return (
+            <span
+              key={`${word.text}-${word.startTime}`}
+              style={{
+                fontSize: 48,
+                fontWeight: isActive ? 'bold' : 'normal',
+                color: isActive ? '#ffcc00' : isPast ? '#888888' : '#ffffff',
+                fontFamily: 'sans-serif',
+                textShadow: '2px 2px 4px rgba(0,0,0,0.8)',
+                transform: isActive ? 'scale(1.1)' : 'scale(1)',
+                transition: 'all 0.1s ease',
+              }}
+            >
+              {word.text}
+            </span>
+          );
+        })}
+      </div>
+    </div>
+  );
+};
+```
+
+## TikTok-Style Word Highlighting
+
+```tsx
+// src/components/TikTokCaptions.tsx
+import { useCurrentFrame, useVideoConfig, spring, interpolate } from 'remotion';
+
+interface Word {
+  text: string;
+  startTime: number;
+  endTime: number;
+}
+
+interface TikTokCaptionsProps {
+  words: Word[];
+  wordsPerPage?: number;
+}
+
+export const TikTokCaptions: React.FC<TikTokCaptionsProps> = ({
+  words,
+  wordsPerPage = 4,
+}) => {
+  const frame = useCurrentFrame();
+  const { fps } = useVideoConfig();
+
+  const currentTime = frame / fps;
+
+  // Find current word index
+  const currentWordIndex = words.findIndex(
+    (word) => currentTime >= word.startTime && currentTime <= word.endTime
+  );
+
+  // Calculate current page
+  const currentPage = Math.floor(Math.max(0, currentWordIndex) / wordsPerPage);
+  const pageStartIndex = currentPage * wordsPerPage;
+  const pageWords = words.slice(pageStartIndex, pageStartIndex + wordsPerPage);
+
+  return (
+    <div
+      style={{
+        position: 'absolute',
+        bottom: 200,
+        left: 0,
+        right: 0,
+        display: 'flex',
+        flexDirection: 'column',
+        alignItems: 'center',
+      }}
+    >
+      <div
+        style={{
+          display: 'flex',
+          flexWrap: 'wrap',
+          justifyContent: 'center',
+          gap: '12px 16px',
+          maxWidth: '80%',
+        }}
+      >
+        {pageWords.map((word, idx) => {
+          const absoluteIndex = pageStartIndex + idx;
+          const isActive = absoluteIndex === currentWordIndex;
+          const isPast =
+            absoluteIndex < currentWordIndex ||
+            (absoluteIndex === currentWordIndex && currentTime > word.endTime);
+
+          // Animation for active word
+          const scale = isActive
+            ? spring({
+                frame: frame - word.startTime * fps,
+                fps,
+                config: { damping: 15, stiffness: 200 },
+                durationInFrames: 10,
+              })
+            : 1;
+
+          const animatedScale = interpolate(scale, [0, 1], [0.8, 1.15]);
+
+          return (
+            <span
+              key={`${word.text}-${absoluteIndex}`}
+              style={{
+                fontSize: 64,
+                fontWeight: 800,
+                fontFamily: 'Inter, sans-serif',
+                color: isActive ? '#FFFFFF' : isPast ? '#666666' : '#AAAAAA',
+                backgroundColor: isActive ? '#FF0050' : 'transparent',
+                padding: isActive ? '4px 16px' : '4px 8px',
+                borderRadius: 8,
+                transform: `scale(${isActive ? animatedScale : 1})`,
+                textShadow: '2px 2px 8px rgba(0,0,0,0.5)',
+              }}
+            >
+              {word.text}
+            </span>
+          );
+        })}
+      </div>
+    </div>
+  );
+};
+```
+
+## Complete Integration Example
+
+```tsx
+// src/compositions/VideoWithCaptions.tsx
+import { Audio } from '@remotion/media';
+import { AbsoluteFill, staticFile } from 'remotion';
+import { TikTokCaptions } from '../components/TikTokCaptions';
+
+interface Props {
+  audioPath: string;
+  words: Array<{
+    text: string;
+    startTime: number;
+    endTime: number;
+  }>;
+}
+
+export const VideoWithCaptions: React.FC<Props> = ({ audioPath, words }) => {
+  return (
+    <AbsoluteFill style={{ backgroundColor: '#1a1a2e' }}>
+      {/* Background or video content */}
+      <AbsoluteFill
+        style={{
+          background: 'linear-gradient(135deg, #1a1a2e 0%, #16213e 100%)',
+        }}
+      />
+
+      {/* Captions */}
+      <TikTokCaptions words={words} wordsPerPage={4} />
+
+      {/* Audio */}
+      <Audio src={staticFile(audioPath)} />
+    </AbsoluteFill>
+  );
+};
+```
+
+## Export to SRT Format
+
+```typescript
+// src/resemble/captions.ts
+export function wordsToSRT(words: Word[], wordsPerCaption: number = 6): string {
+  const lines: string[] = [];
+  let captionIndex = 1;
+
+  for (let i = 0; i < words.length; i += wordsPerCaption) {
+    const chunk = words.slice(i, i + wordsPerCaption);
+    const startTime = chunk[0].startTime;
+    const endTime = chunk[chunk.length - 1].endTime;
+    const text = chunk.map((w) => w.text).join(' ');
+
+    lines.push(`${captionIndex}`);
+    lines.push(`${formatSRTTime(startTime)} --> ${formatSRTTime(endTime)}`);
+    lines.push(text);
+    lines.push('');
+
+    captionIndex++;
+  }
+
+  return lines.join('\n');
+}
+
+function formatSRTTime(seconds: number): string {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+  const ms = Math.round((seconds % 1) * 1000);
+
+  return `${pad(hours)}:${pad(minutes)}:${pad(secs)},${pad(ms, 3)}`;
+}
+
+function pad(num: number, size: number = 2): string {
+  return num.toString().padStart(size, '0');
+}
+```

package/skills/remotion-resemble-ai/rules/resemble-remotion-integration.md

@@ -0,0 +1,305 @@
+---
+name: resemble-remotion-integration
+description: Integrating Resemble.ai generated audio into Remotion compositions
+metadata:
+  tags: resemble, remotion, audio, integration, composition, video
+---
+
+# Integrating Resemble.ai with Remotion
+
+This guide shows how to combine Resemble.ai generated voiceovers with Remotion video compositions.
+
+> **Note:** By default, generate audio only without processing timestamps. Captions/word timestamps are an opt-in feature - only implement them when the user explicitly requests captions, subtitles, or word highlighting.
+
+## Prerequisites
+
+Install Remotion media package:
+
+```bash
+npx remotion add @remotion/media
+```
+
+## Basic Integration
+
+### 1. Generate Audio File
+
+First, generate your voiceover using the sync API:
+
+```typescript
+// scripts/generate-audio.ts
+import 'dotenv/config';
+import { join } from 'path';
+import { generateAudioFile } from '../src/resemble/synthesize';
+
+const VOICE_UUID = process.env.RESEMBLE_VOICE_UUID!;
+
+async function generateVoiceover() {
+  const script = 'Welcome to our product demo. Let me show you around.';
+
+  const { filePath, duration, timestamps } = await generateAudioFile(
+    script,
+    VOICE_UUID,
+    join(process.cwd(), 'public', 'audio', 'voiceover.wav')
+  );
+
+  // Save metadata for use in Remotion
+  const metadata = {
+    audioPath: 'audio/voiceover.wav',
+    duration,
+    fps: 30,
+    durationInFrames: Math.ceil(duration * 30),
+  };
+
+  const fs = await import('fs');
+  fs.writeFileSync(
+    join(process.cwd(), 'src', 'video-config.json'),
+    JSON.stringify(metadata, null, 2)
+  );
+
+  console.log('Generated audio and metadata');
+}
+
+generateVoiceover().catch(console.error);
+```
+
+### 2. Create Remotion Composition
+
+```tsx
+// src/compositions/VoiceoverDemo.tsx
+import { Audio } from '@remotion/media';
+import { AbsoluteFill, staticFile, useCurrentFrame, useVideoConfig } from 'remotion';
+
+interface VoiceoverDemoProps {
+  audioPath: string;
+}
+
+export const VoiceoverDemo: React.FC<VoiceoverDemoProps> = ({ audioPath }) => {
+  const frame = useCurrentFrame();
+  const { fps } = useVideoConfig();
+
+  return (
+    <AbsoluteFill style={{ backgroundColor: '#1a1a2e' }}>
+      {/* Your visual content */}
+      <AbsoluteFill
+        style={{
+          justifyContent: 'center',
+          alignItems: 'center',
+        }}
+      >
+        <h1
+          style={{
+            color: 'white',
+            fontSize: 72,
+            fontFamily: 'sans-serif',
+          }}
+        >
+          Product Demo
+        </h1>
+      </AbsoluteFill>
+
+      {/* Resemble.ai generated audio */}
+      <Audio src={staticFile(audioPath)} />
+    </AbsoluteFill>
+  );
+};
+```
+
+### 3. Register Composition
+
+```tsx
+// src/Root.tsx
+import { Composition } from 'remotion';
+import { VoiceoverDemo } from './compositions/VoiceoverDemo';
+
+// Import generated metadata
+import videoConfig from './video-config.json';
+
+export const RemotionRoot: React.FC = () => {
+  return (
+    <>
+      <Composition
+        id="VoiceoverDemo"
+        component={VoiceoverDemo}
+        durationInFrames={videoConfig.durationInFrames}
+        fps={videoConfig.fps}
+        width={1920}
+        height={1080}
+        defaultProps={{
+          audioPath: videoConfig.audioPath,
+        }}
+      />
+    </>
+  );
+};
+```
+
+## Dynamic Duration Based on Audio
+
+Use `calculateMetadata` to set composition duration based on audio:
+
+```tsx
+// src/compositions/DynamicVoiceover.tsx
+import { Audio } from '@remotion/media';
+import { AbsoluteFill, staticFile, CalculateMetadataFunction } from 'remotion';
+import { getAudioDurationInSeconds } from '@remotion/media';
+
+interface Props {
+  script: string;
+  audioPath: string;
+}
+
+export const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  // Get audio duration from generated file
+  const audioDuration = await getAudioDurationInSeconds(
+    staticFile(props.audioPath)
+  );
+
+  const fps = 30;
+
+  return {
+    fps,
+    durationInFrames: Math.ceil(audioDuration * fps) + 30, // Add 1 second padding
+    width: 1920,
+    height: 1080,
+  };
+};
+
+export const DynamicVoiceover: React.FC<Props> = ({ audioPath }) => {
+  return (
+    <AbsoluteFill>
+      {/* Visual content */}
+      <Audio src={staticFile(audioPath)} />
+    </AbsoluteFill>
+  );
+};
+```
+
+## Multi-Segment Composition
+
+For videos with multiple voiceover segments:
+
+```tsx
+// src/compositions/MultiSegmentVideo.tsx
+import { Audio } from '@remotion/media';
+import {
+  AbsoluteFill,
+  Sequence,
+  staticFile,
+  useVideoConfig,
+} from 'remotion';
+
+interface Segment {
+  id: string;
+  audioPath: string;
+  startFrame: number;
+  durationFrames: number;
+  title: string;
+}
+
+interface Props {
+  segments: Segment[];
+}
+
+export const MultiSegmentVideo: React.FC<Props> = ({ segments }) => {
+  const { fps } = useVideoConfig();
+
+  return (
+    <AbsoluteFill style={{ backgroundColor: '#0f0f23' }}>
+      {segments.map((segment) => (
+        <Sequence
+          key={segment.id}
+          from={segment.startFrame}
+          durationInFrames={segment.durationFrames}
+        >
+          <AbsoluteFill
+            style={{
+              justifyContent: 'center',
+              alignItems: 'center',
+            }}
+          >
+            <h2
+              style={{
+                color: 'white',
+                fontSize: 48,
+                fontFamily: 'sans-serif',
+              }}
+            >
+              {segment.title}
+            </h2>
+          </AbsoluteFill>
+
+          <Audio src={staticFile(segment.audioPath)} />
+        </Sequence>
+      ))}
+    </AbsoluteFill>
+  );
+};
+```
+
+## Audio with Volume Control
+
+Fade in/out the voiceover:
+
+```tsx
+import { Audio } from '@remotion/media';
+import { interpolate, useCurrentFrame, useVideoConfig, staticFile } from 'remotion';
+
+export const VoiceoverWithFade: React.FC<{ audioPath: string }> = ({ audioPath }) => {
+  const frame = useCurrentFrame();
+  const { fps, durationInFrames } = useVideoConfig();
+
+  // Fade in over first second, fade out over last second
+  const volume = interpolate(
+    frame,
+    [0, fps, durationInFrames - fps, durationInFrames],
+    [0, 1, 1, 0],
+    { extrapolateLeft: 'clamp', extrapolateRight: 'clamp' }
+  );
+
+  return <Audio src={staticFile(audioPath)} volume={volume} />;
+};
+```
+
+## Combining with Background Music
+
+```tsx
+import { Audio } from '@remotion/media';
+import { staticFile, AbsoluteFill } from 'remotion';
+
+interface Props {
+  voiceoverPath: string;
+  musicPath: string;
+}
+
+export const VoiceoverWithMusic: React.FC<Props> = ({
+  voiceoverPath,
+  musicPath,
+}) => {
+  return (
+    <AbsoluteFill>
+      {/* Background music at lower volume */}
+      <Audio src={staticFile(musicPath)} volume={0.2} loop />
+
+      {/* Voiceover at full volume */}
+      <Audio src={staticFile(voiceoverPath)} volume={1} />
+
+      {/* Visual content */}
+    </AbsoluteFill>
+  );
+};
+```
+
+## Rendering the Final Video
+
+```bash
+# Preview in browser
+npx remotion studio
+
+# Render to file
+npx remotion render VoiceoverDemo out/video.mp4
+
+# Render with specific codec
+npx remotion render VoiceoverDemo out/video.mp4 --codec h264
+```