cue-ai 0.3.0

package/resources/skills/skills/design/remotion-best-practices/rules/transcribe-captions.md

@@ -0,0 +1,70 @@
+---
+name: transcribe-captions
+description: Transcribing audio to generate captions in Remotion
+metadata:
+  tags: captions, transcribe, whisper, audio, speech-to-text
+---
+
+# Transcribing audio
+
+To transcribe audio to generate captions in Remotion, you can use the [`transcribe()`](https://www.remotion.dev/docs/install-whisper-cpp/transcribe) function from the [`@remotion/install-whisper-cpp`](https://www.remotion.dev/docs/install-whisper-cpp) package.
+
+## Prerequisites
+
+First, the @remotion/install-whisper-cpp package needs to be installed.
+If it is not installed, use the following command:
+
+```bash
+npx remotion add @remotion/install-whisper-cpp
+```
+
+## Transcribing
+
+Make a Node.js script to download Whisper.cpp and a model, and transcribe the audio.
+
+```ts
+import path from "path";
+import {
+  downloadWhisperModel,
+  installWhisperCpp,
+  transcribe,
+  toCaptions,
+} from "@remotion/install-whisper-cpp";
+import fs from "fs";
+
+const to = path.join(process.cwd(), "whisper.cpp");
+
+await installWhisperCpp({
+  to,
+  version: "1.5.5",
+});
+
+await downloadWhisperModel({
+  model: "medium.en",
+  folder: to,
+});
+
+// Convert the audio to a 16KHz wav file first if needed:
+// import {execSync} from 'child_process';
+// execSync('ffmpeg -i /path/to/audio.mp4 -ar 16000 /path/to/audio.wav -y');
+
+const whisperCppOutput = await transcribe({
+  model: "medium.en",
+  whisperPath: to,
+  whisperCppVersion: "1.5.5",
+  inputPath: "/path/to/audio123.wav",
+  tokenLevelTimestamps: true,
+});
+
+// Optional: Apply our recommended postprocessing
+const { captions } = toCaptions({
+  whisperCppOutput,
+});
+
+// Write it to the public/ folder so it can be fetched from Remotion
+fs.writeFileSync("captions123.json", JSON.stringify(captions, null, 2));
+```
+
+Transcribe each clip individually and create multiple JSON files.
+
+See [Displaying captions](display-captions.md) for how to display the captions in Remotion.

package/resources/skills/skills/design/remotion-best-practices/rules/transitions.md

@@ -0,0 +1,197 @@
+---
+name: transitions
+description: Scene transitions and overlays for Remotion using TransitionSeries.
+metadata:
+  tags: transitions, overlays, fade, slide, wipe, scenes
+---
+
+## TransitionSeries
+
+`<TransitionSeries>` arranges scenes and supports two ways to enhance the cut point between them:
+
+- **Transitions** (`<TransitionSeries.Transition>`) — crossfade, slide, wipe, etc. between two scenes. Shortens the timeline because both scenes play simultaneously during the transition.
+- **Overlays** (`<TransitionSeries.Overlay>`) — render an effect (e.g. a light leak) on top of the cut point without shortening the timeline.
+
+Children are absolutely positioned.
+
+## Prerequisites
+
+```bash
+npx remotion add @remotion/transitions
+```
+
+## Transition example
+
+```tsx
+import { TransitionSeries, linearTiming } from "@remotion/transitions";
+import { fade } from "@remotion/transitions/fade";
+
+<TransitionSeries>
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneA />
+  </TransitionSeries.Sequence>
+  <TransitionSeries.Transition
+    presentation={fade()}
+    timing={linearTiming({ durationInFrames: 15 })}
+  />
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneB />
+  </TransitionSeries.Sequence>
+</TransitionSeries>;
+```
+
+## Overlay example
+
+Any React component can be used as an overlay. For a ready-made effect, see the **light-leaks** rule.
+
+```tsx
+import { TransitionSeries } from "@remotion/transitions";
+import { LightLeak } from "@remotion/light-leaks";
+
+<TransitionSeries>
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneA />
+  </TransitionSeries.Sequence>
+  <TransitionSeries.Overlay durationInFrames={20}>
+    <LightLeak />
+  </TransitionSeries.Overlay>
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneB />
+  </TransitionSeries.Sequence>
+</TransitionSeries>;
+```
+
+## Mixing transitions and overlays
+
+Transitions and overlays can coexist in the same `<TransitionSeries>`, but an overlay cannot be adjacent to a transition or another overlay.
+
+```tsx
+import { TransitionSeries, linearTiming } from "@remotion/transitions";
+import { fade } from "@remotion/transitions/fade";
+import { LightLeak } from "@remotion/light-leaks";
+
+<TransitionSeries>
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneA />
+  </TransitionSeries.Sequence>
+  <TransitionSeries.Overlay durationInFrames={30}>
+    <LightLeak />
+  </TransitionSeries.Overlay>
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneB />
+  </TransitionSeries.Sequence>
+  <TransitionSeries.Transition
+    presentation={fade()}
+    timing={linearTiming({ durationInFrames: 15 })}
+  />
+  <TransitionSeries.Sequence durationInFrames={60}>
+    <SceneC />
+  </TransitionSeries.Sequence>
+</TransitionSeries>;
+```
+
+## Transition props
+
+`<TransitionSeries.Transition>` requires:
+
+- `presentation` — the visual effect (e.g. `fade()`, `slide()`, `wipe()`).
+- `timing` — controls speed and easing (e.g. `linearTiming()`, `springTiming()`).
+
+## Overlay props
+
+`<TransitionSeries.Overlay>` accepts:
+
+- `durationInFrames` — how long the overlay is visible (positive integer).
+- `offset?` — shifts the overlay relative to the cut point center. Positive = later, negative = earlier. Default: `0`.
+
+## Available transition types
+
+Import transitions from their respective modules:
+
+```tsx
+import { fade } from "@remotion/transitions/fade";
+import { slide } from "@remotion/transitions/slide";
+import { wipe } from "@remotion/transitions/wipe";
+import { flip } from "@remotion/transitions/flip";
+import { clockWipe } from "@remotion/transitions/clock-wipe";
+```
+
+## Slide transition with direction
+
+```tsx
+import { slide } from "@remotion/transitions/slide";
+
+<TransitionSeries.Transition
+  presentation={slide({ direction: "from-left" })}
+  timing={linearTiming({ durationInFrames: 20 })}
+/>;
+```
+
+Directions: `"from-left"`, `"from-right"`, `"from-top"`, `"from-bottom"`
+
+## Timing options
+
+```tsx
+import { linearTiming, springTiming } from "@remotion/transitions";
+
+// Linear timing - constant speed
+linearTiming({ durationInFrames: 20 });
+
+// Spring timing - organic motion
+springTiming({ config: { damping: 200 }, durationInFrames: 25 });
+```
+
+## Duration calculation
+
+Transitions overlap adjacent scenes, so the total composition length is **shorter** than the sum of all sequence durations. Overlays do **not** affect the total duration.
+
+For example, with two 60-frame sequences and a 15-frame transition:
+
+- Without transitions: `60 + 60 = 120` frames
+- With transition: `60 + 60 - 15 = 105` frames
+
+Adding an overlay between two other sequences does not change the total.
+
+### Getting the duration of a transition
+
+Use the `getDurationInFrames()` method on the timing object:
+
+```tsx
+import { linearTiming, springTiming } from "@remotion/transitions";
+
+const linearDuration = linearTiming({
+  durationInFrames: 20,
+}).getDurationInFrames({ fps: 30 });
+// Returns 20
+
+const springDuration = springTiming({
+  config: { damping: 200 },
+}).getDurationInFrames({ fps: 30 });
+// Returns calculated duration based on spring physics
+```
+
+For `springTiming` without an explicit `durationInFrames`, the duration depends on `fps` because it calculates when the spring animation settles.
+
+### Calculating total composition duration
+
+```tsx
+import { linearTiming } from "@remotion/transitions";
+
+const scene1Duration = 60;
+const scene2Duration = 60;
+const scene3Duration = 60;
+
+const timing1 = linearTiming({ durationInFrames: 15 });
+const timing2 = linearTiming({ durationInFrames: 20 });
+
+const transition1Duration = timing1.getDurationInFrames({ fps: 30 });
+const transition2Duration = timing2.getDurationInFrames({ fps: 30 });
+
+const totalDuration =
+  scene1Duration +
+  scene2Duration +
+  scene3Duration -
+  transition1Duration -
+  transition2Duration;
+// 60 + 60 + 60 - 15 - 20 = 145 frames
+```

package/resources/skills/skills/design/remotion-best-practices/rules/transparent-videos.md

@@ -0,0 +1,106 @@
+---
+name: transparent-videos
+description: Rendering transparent videos in Remotion
+metadata:
+  tags: transparent, alpha, codec, vp9, prores, webm
+---
+
+# Rendering Transparent Videos
+
+Remotion can render transparent videos in two ways: as a ProRes video or as a WebM video.
+
+## Transparent ProRes
+
+Ideal for when importing into video editing software.
+
+**CLI:**
+
+```bash
+npx remotion render --image-format=png --pixel-format=yuva444p10le --codec=prores --prores-profile=4444 MyComp out.mov
+```
+
+**Default in Studio** (restart Studio after changing):
+
+```ts
+// remotion.config.ts
+import { Config } from "@remotion/cli/config";
+
+Config.setVideoImageFormat("png");
+Config.setPixelFormat("yuva444p10le");
+Config.setCodec("prores");
+Config.setProResProfile("4444");
+```
+
+**Setting it as the default export settings for a composition** (using `calculateMetadata`):
+
+```tsx
+import { CalculateMetadataFunction } from "remotion";
+
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  return {
+    defaultCodec: "prores",
+    defaultVideoImageFormat: "png",
+    defaultPixelFormat: "yuva444p10le",
+    defaultProResProfile: "4444",
+  };
+};
+
+<Composition
+  id="my-video"
+  component={MyVideo}
+  durationInFrames={150}
+  fps={30}
+  width={1920}
+  height={1080}
+  calculateMetadata={calculateMetadata}
+/>;
+```
+
+## Transparent WebM (VP9)
+
+Ideal for when playing in a browser.
+
+**CLI:**
+
+```bash
+npx remotion render --image-format=png --pixel-format=yuva420p --codec=vp9 MyComp out.webm
+```
+
+**Default in Studio** (restart Studio after changing):
+
+```ts
+// remotion.config.ts
+import { Config } from "@remotion/cli/config";
+
+Config.setVideoImageFormat("png");
+Config.setPixelFormat("yuva420p");
+Config.setCodec("vp9");
+```
+
+**Setting it as the default export settings for a composition** (using `calculateMetadata`):
+
+```tsx
+import { CalculateMetadataFunction } from "remotion";
+
+const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  return {
+    defaultCodec: "vp8",
+    defaultVideoImageFormat: "png",
+    defaultPixelFormat: "yuva420p",
+  };
+};
+
+<Composition
+  id="my-video"
+  component={MyVideo}
+  durationInFrames={150}
+  fps={30}
+  width={1920}
+  height={1080}
+  calculateMetadata={calculateMetadata}
+/>;
+```

package/resources/skills/skills/design/remotion-best-practices/rules/trimming.md

@@ -0,0 +1,51 @@
+---
+name: trimming
+description: Trimming patterns for Remotion - cut the beginning or end of animations
+metadata:
+  tags: sequence, trim, clip, cut, offset
+---
+
+Use `<Sequence>` with a negative `from` value to trim the start of an animation.
+
+## Trim the Beginning
+
+A negative `from` value shifts time backwards, making the animation start partway through:
+
+```tsx
+import { Sequence, useVideoConfig } from "remotion";
+
+const fps = useVideoConfig();
+
+<Sequence from={-0.5 * fps}>
+  <MyAnimation />
+</Sequence>;
+```
+
+The animation appears 15 frames into its progress - the first 15 frames are trimmed off.
+Inside `<MyAnimation>`, `useCurrentFrame()` starts at 15 instead of 0.
+
+## Trim the End
+
+Use `durationInFrames` to unmount content after a specified duration:
+
+```tsx
+<Sequence durationInFrames={1.5 * fps}>
+  <MyAnimation />
+</Sequence>
+```
+
+The animation plays for 45 frames, then the component unmounts.
+
+## Trim and Delay
+
+Nest sequences to both trim the beginning and delay when it appears:
+
+```tsx
+<Sequence from={30}>
+  <Sequence from={-15}>
+    <MyAnimation />
+  </Sequence>
+</Sequence>
+```
+
+The inner sequence trims 15 frames from the start, and the outer sequence delays the result by 30 frames.

package/resources/skills/skills/design/remotion-best-practices/rules/videos.md

@@ -0,0 +1,171 @@
+---
+name: videos
+description: Embedding videos in Remotion - trimming, volume, speed, looping, pitch
+metadata:
+  tags: video, media, trim, volume, speed, loop, pitch
+---
+
+# Using videos in Remotion
+
+## Prerequisites
+
+First, the @remotion/media package needs to be installed.  
+If it is not, use the following command:
+
+```bash
+npx remotion add @remotion/media # If project uses npm
+bunx remotion add @remotion/media # If project uses bun
+yarn remotion add @remotion/media # If project uses yarn
+pnpm exec remotion add @remotion/media # If project uses pnpm
+```
+
+Use `<Video>` from `@remotion/media` to embed videos into your composition.
+
+```tsx
+import { Video } from "@remotion/media";
+import { staticFile } from "remotion";
+
+export const MyComposition = () => {
+  return <Video src={staticFile("video.mp4")} />;
+};
+```
+
+Remote URLs are also supported:
+
+```tsx
+<Video src="https://remotion.media/video.mp4" />
+```
+
+## Trimming
+
+Use `trimBefore` and `trimAfter` to remove portions of the video. Values are in seconds.
+
+```tsx
+const { fps } = useVideoConfig();
+
+return (
+  <Video
+    src={staticFile("video.mp4")}
+    trimBefore={2 * fps} // Skip the first 2 seconds
+    trimAfter={10 * fps} // End at the 10 second mark
+  />
+);
+```
+
+## Delaying
+
+Wrap the video in a `<Sequence>` to delay when it appears:
+
+```tsx
+import { Sequence, staticFile } from "remotion";
+import { Video } from "@remotion/media";
+
+const { fps } = useVideoConfig();
+
+return (
+  <Sequence from={1 * fps}>
+    <Video src={staticFile("video.mp4")} />
+  </Sequence>
+);
+```
+
+The video will appear after 1 second.
+
+## Sizing and Position
+
+Use the `style` prop to control size and position:
+
+```tsx
+<Video
+  src={staticFile("video.mp4")}
+  style={{
+    width: 500,
+    height: 300,
+    position: "absolute",
+    top: 100,
+    left: 50,
+    objectFit: "cover",
+  }}
+/>
+```
+
+## Volume
+
+Set a static volume (0 to 1):
+
+```tsx
+<Video src={staticFile("video.mp4")} volume={0.5} />
+```
+
+Or use a callback for dynamic volume based on the current frame:
+
+```tsx
+import { interpolate } from "remotion";
+
+const { fps } = useVideoConfig();
+
+return (
+  <Video
+    src={staticFile("video.mp4")}
+    volume={(f) =>
+      interpolate(f, [0, 1 * fps], [0, 1], { extrapolateRight: "clamp" })
+    }
+  />
+);
+```
+
+Use `muted` to silence the video entirely:
+
+```tsx
+<Video src={staticFile("video.mp4")} muted />
+```
+
+## Speed
+
+Use `playbackRate` to change the playback speed:
+
+```tsx
+<Video src={staticFile("video.mp4")} playbackRate={2} /> {/* 2x speed */}
+<Video src={staticFile("video.mp4")} playbackRate={0.5} /> {/* Half speed */}
+```
+
+Reverse playback is not supported.
+
+## Looping
+
+Use `loop` to loop the video indefinitely:
+
+```tsx
+<Video src={staticFile("video.mp4")} loop />
+```
+
+Use `loopVolumeCurveBehavior` to control how the frame count behaves when looping:
+
+- `"repeat"`: Frame count resets to 0 each loop (for `volume` callback)
+- `"extend"`: Frame count continues incrementing
+
+```tsx
+<Video
+  src={staticFile("video.mp4")}
+  loop
+  loopVolumeCurveBehavior="extend"
+  volume={(f) => interpolate(f, [0, 300], [1, 0])} // Fade out over multiple loops
+/>
+```
+
+## Pitch
+
+Use `toneFrequency` to adjust the pitch without affecting speed. Values range from 0.01 to 2:
+
+```tsx
+<Video
+  src={staticFile("video.mp4")}
+  toneFrequency={1.5} // Higher pitch
+/>
+<Video
+  src={staticFile("video.mp4")}
+  toneFrequency={0.8} // Lower pitch
+/>
+```
+
+Pitch shifting only works during server-side rendering, not in the Remotion Studio preview or in the `<Player />`.

package/resources/skills/skills/design/remotion-best-practices/rules/voiceover.md

@@ -0,0 +1,99 @@
+---
+name: voiceover
+description: Adding AI-generated voiceover to Remotion compositions using TTS
+metadata:
+  tags: voiceover, audio, elevenlabs, tts, speech, calculateMetadata, dynamic duration
+---
+
+# Adding AI voiceover to a Remotion composition
+
+Use ElevenLabs TTS to generate speech audio per scene, then use [`calculateMetadata`](./calculate-metadata) to dynamically size the composition to match the audio.
+
+## Prerequisites
+
+By default this guide uses **ElevenLabs** as the TTS provider (`ELEVENLABS_API_KEY` environment variable). Users may substitute any TTS service that can produce an audio file.
+
+If the user has not specified a TTS provider, recommend ElevenLabs and ask for their API key.
+
+Ensure the environment variable is available when running the generation script:
+
+```bash
+node --strip-types generate-voiceover.ts
+```
+
+## Generating audio with ElevenLabs
+
+Create a script that reads the config, calls the ElevenLabs API for each scene, and writes MP3 files to the `public/` directory so Remotion can access them via `staticFile()`.
+
+The core API call for a single scene:
+
+```ts title="generate-voiceover.ts"
+const response = await fetch(
+  `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+  {
+    method: "POST",
+    headers: {
+      "xi-api-key": process.env.ELEVENLABS_API_KEY!,
+      "Content-Type": "application/json",
+      Accept: "audio/mpeg",
+    },
+    body: JSON.stringify({
+      text: "Welcome to the show.",
+      model_id: "eleven_multilingual_v2",
+      voice_settings: {
+        stability: 0.5,
+        similarity_boost: 0.75,
+        style: 0.3,
+      },
+    }),
+  },
+);
+
+const audioBuffer = Buffer.from(await response.arrayBuffer());
+writeFileSync(`public/voiceover/${compositionId}/${scene.id}.mp3`, audioBuffer);
+```
+
+## Dynamic composition duration with calculateMetadata
+
+Use [`calculateMetadata`](./calculate-metadata.md) to measure the [audio durations](./get-audio-duration.md) and set the composition length accordingly.
+
+```tsx
+import { CalculateMetadataFunction, staticFile } from "remotion";
+import { getAudioDuration } from "./get-audio-duration";
+
+const FPS = 30;
+
+const SCENE_AUDIO_FILES = [
+  "voiceover/my-comp/scene-01-intro.mp3",
+  "voiceover/my-comp/scene-02-main.mp3",
+  "voiceover/my-comp/scene-03-outro.mp3",
+];
+
+export const calculateMetadata: CalculateMetadataFunction<Props> = async ({
+  props,
+}) => {
+  const durations = await Promise.all(
+    SCENE_AUDIO_FILES.map((file) => getAudioDuration(staticFile(file))),
+  );
+
+  const sceneDurations = durations.map((durationInSeconds) => {
+    return durationInSeconds * FPS;
+  });
+
+  return {
+    durationInFrames: Math.ceil(sceneDurations.reduce((sum, d) => sum + d, 0)),
+  };
+};
+```
+
+The computed `sceneDurations` are passed into the component via a `voiceover` prop so the component knows how long each scene should be.
+
+If the composition uses [`<TransitionSeries>`](./transitions.md), subtract the overlap from total duration: [./transitions.md#calculating-total-composition-duration](./transitions.md#calculating-total-composition-duration)
+
+## Rendering audio in the component
+
+See [audio.md](./audio.md) for more information on how to render audio in the component.
+
+## Delaying audio start
+
+See [audio.md#delaying](./audio.md#delaying) for more information on how to delay the audio start.