@damiangilgonzalez/harness 1.3.7

package/skills/remotion-video/evals/cases.yaml

@@ -0,0 +1,64 @@
+skill: remotion-video
+
+# Prompts that MUST load `remotion-video`. The skill owns the ENCODE: turning React
+# code + assets into a real MP4/MOV via Remotion — the composition/Sequence/
+# TransitionSeries graph, the Whisper.cpp → toCaptions → createTikTokStyleCaptions
+# captions pipeline, the upstream silence pass, b-roll overlays, and `npx remotion render`.
+# It does NOT write the script/hook/caption text (video-shorts), master audio or build
+# an RSS feed (podcast), structure narrative (course-storytelling), design the thumbnail
+# image (youtube-thumbnails), or write title/description/tags (youtube-packaging).
+should_trigger:
+  - prompt: "Render my script and voiceover into an MP4 with Remotion."
+    why: "Core ask — producing the actual file from code + assets is exactly this skill's deliverable."
+
+  - prompt: "Burn animated word-by-word captions onto this video from the Whisper transcript."
+    why: "Non-obvious — names a transcript and burned captions; this is the toCaptions → createTikTokStyleCaptions pipeline, not caption copywriting."
+
+  - prompt: "Automatically remove the silences from my recording before I render it."
+    why: "Non-obvious — the silence pass (auto-editor/ffmpeg) sits upstream of Remotion and is owned here, not by a script skill."
+
+  - prompt: "Add slide and fade transitions between scenes in my Remotion composition."
+    why: "Programmatic transitions via @remotion/transitions <TransitionSeries> — pixels and frames, this skill."
+
+  - prompt: "Quita los silencios y genera el vídeo final con subtítulos quemados."
+    why: "Spanish for 'remove the silences and generate the final video with burned-in subtitles' — the full silence-pass + captions + render pipeline."
+
+  - prompt: "Set up a headless Remotion render in CI that outputs a checkable MP4."
+    why: "Non-obvious — CI/headless render with a verifiable artifact is the rendering half of this skill."
+
+  - prompt: "Renderitza la composició a MP4 amb transicions i overlay de b-roll."
+    why: "Catalan for 'render the composition to MP4 with transitions and a b-roll overlay' — composition graph + transitions + overlay encode."
+
+should_not_trigger:
+  - prompt: "Write the hook and shot-by-shot script for a 30-second Reel."
+    route_to: video-shorts
+    why: "The words, beats and caption text are the script craft; this skill only encodes them into frames."
+
+  - prompt: "Master my podcast audio to -16 LUFS and write the RSS item."
+    route_to: podcast
+    why: "Audio mastering to a loudness spec and feed generation are audio-first, not a video render."
+
+  - prompt: "Design a clickable thumbnail image for this video."
+    route_to: youtube-thumbnails
+    why: "A still image design, not a rendered video file."
+
+  - prompt: "Write the YouTube title, description and tags for the upload."
+    route_to: youtube-packaging
+    why: "Metadata copy for the listing, not encoding pixels."
+
+  - prompt: "Structure the narrative arc and lesson flow for my course video."
+    route_to: course-storytelling
+    why: "Narrative architecture decides what story to tell; this skill renders it, it does not structure it."
+
+capability:
+  - scenario: "Given a voiceover MP4, b-roll clips and a transcript, produce a Remotion project that renders a 1080x1920 video with two scenes, a fade transition between them, a b-roll overlay at a timecode, and burned-in TikTok-style captions, then render it."
+    must_include:
+      - "Pinned fps + dimensions on the <Composition> up front (e.g. 1080x1920 @ 30)"
+      - "frames = seconds * fps math applied with Math.round at the edge for any timecode→frame conversion"
+      - "<TransitionSeries> with a presentation (fade) and a timing (linearTiming or springTiming)"
+      - "Captions pipeline: Whisper.cpp → toCaptions → createTikTokStyleCaptions with combineTokensWithinMilliseconds"
+      - "A silence-removal step (auto-editor or ffmpeg) located on the source BEFORE Remotion, not inside the composition"
+      - "B-roll overlay via <OffthreadVideo> inside an <AbsoluteFill> gated by a <Sequence from>"
+      - "`npx remotion render <id> out/video.mp4` with an explicit output path"
+      - "A deterministic-render note (no Date.now()/unseeded Math.random; drive off useCurrentFrame)"
+      - "A short region test (e.g. --frames=0-45) before the full render"

package/skills/remotion-video/references/captions-pipeline.md

@@ -0,0 +1,163 @@
+# Captions pipeline — Whisper.cpp → toCaptions → createTikTokStyleCaptions
+
+The captions layer is built once on a Node server (transcribe + convert) and then
+consumed by a composition at render time. The native `@remotion/captions` package
+shipped in Remotion **v4.0.216** — the same release that deprecated the old
+`convertToCaptions()` helper in favor of `toCaptions()`. This file holds the full
+install, the transcribe step, and the token-highlight renderer component referenced
+from `SKILL.md`.
+
+## 1. Install Whisper.cpp and a model
+
+`@remotion/install-whisper-cpp` downloads the Whisper.cpp binary and a model. The
+`medium.en` model is ~1.5 GB — download it **once** to a cached path and reuse it.
+Never commit it to git.
+
+```ts
+// scripts/install-whisper.ts  (run with: npx tsx scripts/install-whisper.ts)
+import { installWhisperCpp, downloadWhisperModel } from "@remotion/install-whisper-cpp";
+import path from "node:path";
+
+const to = path.join(process.cwd(), "whisper.cpp"); // .gitignore this dir
+
+await installWhisperCpp({ to, version: "1.5.5" });
+await downloadWhisperModel({ folder: to, model: "medium.en" });
+```
+
+```gitignore
+# .gitignore
+whisper.cpp/
+```
+
+## 2. Transcribe audio to captions
+
+The audio must be 16 kHz WAV for Whisper.cpp. Convert with ffmpeg, then transcribe,
+then `toCaptions()` turns the Whisper token stream into a `Caption[]` with
+per-token start/end timestamps.
+
+```ts
+// scripts/transcribe.ts
+import { transcribe, toCaptions } from "@remotion/install-whisper-cpp";
+import path from "node:path";
+import { execSync } from "node:child_process";
+
+const whisperPath = path.join(process.cwd(), "whisper.cpp");
+const src = "public/voiceover.mp4";
+const wav = "public/voiceover.wav";
+
+// Whisper.cpp wants 16kHz mono WAV
+execSync(`npx remotion ffmpeg -i ${src} -ar 16000 ${wav} -y`);
+
+const whisperOutput = await transcribe({
+  inputPath: wav,
+  whisperPath,
+  model: "medium.en",
+  tokenLevelTimestamps: true, // required for word-by-word
+});
+
+const { captions } = toCaptions({ whisperCppOutput: whisperOutput });
+// persist captions.json next to the composition
+import { writeFileSync } from "node:fs";
+writeFileSync("public/captions.json", JSON.stringify(captions, null, 2));
+```
+
+On a pre-v4.0.216 Remotion you will only have the legacy `convertToCaptions()` (from
+`@remotion/install-whisper-cpp/convert-to-captions`) — it produces the same
+`Caption[]` but is deprecated; prefer `toCaptions()` on any current version. A
+`Caption` is roughly `{ text, startMs, endMs, timestampMs, confidence }`.
+
+## 3. Segment into pages
+
+`@remotion/captions` `createTikTokStyleCaptions()` groups tokens into "pages" — the
+chunks that appear on screen together. The `combineTokensWithinMilliseconds` value is
+the page-size dial:
+
+- **~200 ms** → roughly one word per page → word-by-word pop (TikTok energy).
+- **~1200 ms** → a phrase per page → calmer, more readable.
+
+```ts
+import { createTikTokStyleCaptions } from "@remotion/captions";
+
+const { pages } = createTikTokStyleCaptions({
+  captions,                              // the Caption[] from step 2
+  combineTokensWithinMilliseconds: 1200, // phrase pages; drop to ~200 for word-by-word
+});
+```
+
+## 4. The caption renderer component
+
+Render the current page and highlight the active token by comparing each token's
+timestamp against the current time in ms (`frame / fps * 1000`).
+
+```tsx
+// src/Captions.tsx
+import { AbsoluteFill, useCurrentFrame, useVideoConfig } from "remotion";
+import type { TikTokPage } from "@remotion/captions";
+
+export const Captions: React.FC<{ pages: TikTokPage[] }> = ({ pages }) => {
+  const frame = useCurrentFrame();
+  const { fps } = useVideoConfig();
+  const nowMs = (frame / fps) * 1000;
+
+  const page = pages.find((p) => nowMs >= p.startMs && nowMs < p.startMs + p.durInMs());
+  if (!page) return null;
+
+  return (
+    <AbsoluteFill
+      style={{
+        justifyContent: "flex-end",
+        alignItems: "center",
+        // safe zone: keep text clear of the platform UI margins
+        paddingBottom: 320,
+        paddingLeft: 80,
+        paddingRight: 80,
+      }}
+    >
+      <div style={{ display: "flex", flexWrap: "wrap", justifyContent: "center", gap: "0 16px" }}>
+        {page.tokens.map((t, i) => {
+          const active = nowMs >= t.fromMs && nowMs < t.toMs;
+          return (
+            <span
+              key={i}
+              style={{
+                fontFamily: "Inter, sans-serif",
+                fontWeight: 800,
+                fontSize: 72,
+                color: active ? "#FFE600" : "white",
+                // high-contrast stroke so it survives a bright background
+                WebkitTextStroke: "8px black",
+                paintOrder: "stroke fill",
+                transform: active ? "scale(1.06)" : "scale(1)",
+              }}
+            >
+              {t.text}
+            </span>
+          );
+        })}
+      </div>
+    </AbsoluteFill>
+  );
+};
+```
+
+Mount it as the top layer of the composition, above the video and b-roll:
+
+```tsx
+<AbsoluteFill>
+  <OffthreadVideo src={staticFile("voiceover.mp4")} />
+  {/* ...b-roll sequences... */}
+  <Captions pages={pages} />
+</AbsoluteFill>
+```
+
+## Styling and safe-zone notes
+
+- **High contrast + a thick stroke or box.** Captions must read on a bright phone
+  outdoors; a thin font with no stroke vanishes over light footage.
+- **Respect safe zones.** Keep caption text clear of the bottom and right-rail
+  platform UI. For 1080×1920 vertical, a ~320 px bottom inset keeps text above the
+  caption/CTA bar.
+- **One highlight color, consistent.** The active-token color is a brand decision;
+  pick one and reuse it — flicker between colors reads as a glitch.
+- **Page size matches format.** Word-by-word for high-energy shorts; phrase pages for
+  explainers where the viewer is reading, not just feeling rhythm.

package/skills/remotion-video/references/render-and-pipeline.md

@@ -0,0 +1,131 @@
+# Render and pipeline — silence removal, render flags, CI, verification
+
+This file holds the upstream silence pass, the full render flag matrix, headless/CI
+render, and `ffprobe` verification referenced from `SKILL.md`.
+
+## Silence removal (upstream, before Remotion)
+
+The silence pass edits the **source asset**, not a composition. Run it first so every
+downstream frame number already reflects the tightened timeline.
+
+### auto-editor
+
+`auto-editor` is a Python + ffmpeg engine. **pip distribution is stale/discontinued
+— install via the official binary or `pipx`, not `pip install`.**
+
+```bash
+pipx install auto-editor        # or grab the official binary from auto-editor.com
+```
+
+First pass — cut dead space by audio loudness:
+
+```bash
+auto-editor input.mp4 \
+  --margin 0.2s \                  # pad each kept region so cuts don't clip speech
+  --edit audio:threshold=4% \      # loudness floor below which a region is "silence"
+  -o tightened.mp4
+```
+
+Other useful flags:
+
+- `--cut-out 0,30` — force-remove a frame/time range regardless of loudness.
+- `--export premiere` — emit an EDL/XML for an NLE re-import instead of a rendered file
+  (also `resolve`, `final-cut-pro`, `shotcut`).
+- `--edit motion:threshold=2%` — cut by visual stillness instead of audio.
+
+### ffmpeg fallback (no auto-editor)
+
+Detect silence to inspect where the gaps are:
+
+```bash
+ffmpeg -i input.mp4 -af silencedetect=noise=-30dB:d=0.5 -f null - 2>&1 | grep silence_
+```
+
+Remove silence inline with the `silenceremove` filter (trim leading + internal gaps):
+
+```bash
+ffmpeg -i input.mp4 \
+  -af "silenceremove=start_periods=1:start_duration=0.3:start_threshold=-30dB:\
+detection=peak,silenceremove=stop_periods=-1:stop_duration=0.3:stop_threshold=-30dB" \
+  tightened.mp4
+```
+
+`silenceremove` is bluntter than `auto-editor` (it does not pad cuts the same way),
+so prefer `auto-editor` when available and treat ffmpeg as the no-dependency fallback.
+
+## Render flag matrix
+
+```bash
+npx remotion render <CompositionId> [out/video.mp4] [flags]
+```
+
+Omit `<CompositionId>` for an interactive picker. Config lives in `remotion.config.ts`
+via `@remotion/cli/config`; CLI flags override the config file.
+
+```ts
+// remotion.config.ts
+import { Config } from "@remotion/cli/config";
+Config.setConcurrency(8);             // parallel render workers
+Config.setCodec("h264");
+Config.setPixelFormat("yuv420p");     // yuv444p / yuva444p10le for higher-fidelity / alpha
+```
+
+| Goal | Flags |
+|---|---|
+| Web / YouTube default | `--codec=h264` (default) |
+| Smaller file, same quality | `--codec=h265` |
+| Edit-grade master with alpha | `--codec=prores --prores-profile=4444 --pixel-format=yuva444p10le --image-format=png` |
+| Looping preview / web-alpha | `--codec=vp8` or `--codec=gif` |
+| Region test (cheap validation) | `--frames=0-45` |
+| Higher quality (h264/h265) | `--crf=18` (lower CRF = higher quality, bigger file) |
+| Throughput | `--concurrency=8` |
+
+**Region test before every full render:** `npx remotion render MyVideo out/test.mp4
+--frames=0-45`. A full render of a minutes-long composition is expensive; a region
+test surfaces a broken layer in seconds.
+
+## Deterministic output
+
+- Drive all animation off `useCurrentFrame()`; never `Date.now()` or unseeded
+  `Math.random()` — frames render in parallel and on re-render, so non-frame inputs
+  produce different pixels. Use Remotion's `random(seed)` when you need noise.
+- Pin `fps` + `width` + `height` on `<Composition>` and leave them. Changing fps after
+  durations are written silently shifts every frame count.
+- Same input + same code → byte-comparable frames. That is the artifact contract this
+  skill verifies.
+
+## Headless / CI render
+
+CI has no display and no Whisper model cached, so:
+
+1. Cache the Whisper model between runs (it is ~1.5 GB; do not re-download each run,
+   do not commit it).
+2. Install ffmpeg (or rely on `npx remotion ffmpeg`).
+3. Run the render headless and fail the job if the output is empty or has no video
+   stream.
+
+```bash
+npm ci
+npx remotion render MyVideo out/video.mp4 --concurrency=4
+test -s out/video.mp4 || { echo "empty render"; exit 1; }
+ffprobe -v error -select_streams v:0 -show_entries stream=codec_type,width,height \
+  -of csv=p=0 out/video.mp4
+```
+
+## ffprobe verification
+
+Confirm the render is a real video with the dimensions/fps you expect:
+
+```bash
+# video stream present + dimensions
+ffprobe -v error -select_streams v:0 \
+  -show_entries stream=width,height,r_frame_rate,codec_name \
+  -of default=noprint_wrappers=1 out/video.mp4
+
+# duration in seconds
+ffprobe -v error -show_entries format=duration -of csv=p=0 out/video.mp4
+```
+
+`r_frame_rate` reports as a fraction (`30/1` = 30 fps). If `ffprobe` reports no video
+stream, zero dimensions, or the file is empty, the render failed regardless of exit
+code — that is exactly what `scripts/verify.sh` asserts.

package/skills/remotion-video/scripts/verify.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# verify.sh — check that a Remotion project is well-formed and (optionally) renders.
+#
+# Usage:
+#   bash scripts/verify.sh <project-dir>            # read-only: static + `remotion compositions`
+#   bash scripts/verify.sh <project-dir> --render   # also do a short region test render + ffprobe
+#   bash scripts/verify.sh                          # no target -> nothing to check, exit 0
+#
+# Default mode is READ-ONLY: it never renders, installs, or writes. A render produces
+# a file and burns CPU, so the actual test render is gated behind --render (opt-in).
+#
+# What it checks, in the spec's order:
+#   1. If Node + a Remotion project are present: `npx remotion compositions` succeeds
+#      and lists at least one composition id (read-only).
+#   2. With --render AND ffmpeg present: render a short region (frames 0-15) of the
+#      first composition to a temp MP4, assert it is non-empty, and `ffprobe` confirms
+#      a video stream with width/height. The temp file is removed.
+#   3. Static fallback (no node, or no project, or --render not given): assert at least
+#      one composition .tsx exists and a render script/command references a real
+#      composition id + an output path.
+#
+# Exits 0 on a clean project AND on an empty/clean target (no false failure):
+#   - no argument            -> "nothing to check", exit 0
+#   - dir with no .tsx files -> nothing to verify, exit 0
+# Exits non-zero only when a real, present project breaks a rule.
+#
+# Portability: stock macOS bash 3.2. set -u on; set -e off (each check owns its exit).
+
+set -u
+
+if [ -t 1 ] && command -v tput >/dev/null 2>&1; then
+  RED="$(tput setaf 1)"; GREEN="$(tput setaf 2)"; YELLOW="$(tput setaf 3)"; RESET="$(tput sgr0)"
+else
+  RED=""; GREEN=""; YELLOW=""; RESET=""
+fi
+
+fail() { printf '%s\n' "${RED}FAIL: $1${RESET}"; failures=$((failures + 1)); }
+warn() { printf '%s\n' "${YELLOW}WARN: $1${RESET}"; }
+ok()   { printf '%s\n' "${GREEN}OK: $1${RESET}"; }
+
+# --- args ------------------------------------------------------------------
+target=""
+do_render=0
+for arg in "$@"; do
+  case "$arg" in
+    --render) do_render=1 ;;
+    *) [ -z "$target" ] && target="$arg" || warn "ignoring extra argument '$arg'" ;;
+  esac
+done
+
+if [ -z "$target" ]; then
+  printf '%s\n' "${YELLOW}Nothing to check (no project dir given). Pass a Remotion project directory.${RESET}"
+  exit 0
+fi
+
+if [ ! -d "$target" ]; then
+  printf '%s\n' "${YELLOW}'$target' is not a directory. Pass a Remotion project directory. Nothing to check.${RESET}"
+  exit 0
+fi
+
+failures=0
+
+# --- find composition .tsx files (look in target and target/src) -----------
+tsx_files=""
+for d in "$target" "$target/src"; do
+  [ -d "$d" ] || continue
+  for f in "$d"/*.tsx "$d"/**/*.tsx; do
+    [ -e "$f" ] && tsx_files="${tsx_files}${tsx_files:+
+}$f"
+  done
+done
+# de-dup is unnecessary; just detect presence
+has_tsx=0
+[ -n "$tsx_files" ] && has_tsx=1
+
+if [ "$has_tsx" -eq 0 ]; then
+  printf '%s\n' "${YELLOW}No .tsx files under '$target' (or '$target/src'). Not a Remotion project — nothing to verify.${RESET}"
+  exit 0
+fi
+
+printf '\n%s\n' "Checking Remotion project: $target"
+
+# --- extract a composition id from <Composition id="..."> -------------------
+comp_id="$(grep -rhoE 'id[[:space:]]*=[[:space:]]*"[^"]+"' $tsx_files 2>/dev/null \
+  | head -1 | sed -E 's/.*"([^"]+)".*/\1/')"
+
+# =============================================================================
+# Mode 1: dynamic check via the Remotion CLI (read-only)
+# =============================================================================
+ran_dynamic=0
+if command -v node >/dev/null 2>&1 && command -v npx >/dev/null 2>&1 \
+   && [ -f "$target/package.json" ]; then
+  printf '%s\n' "node + package.json present — listing compositions (read-only)..."
+  comps="$(cd "$target" && npx --no-install remotion compositions 2>/dev/null)"
+  if [ -n "$comps" ]; then
+    ran_dynamic=1
+    ok "remotion compositions succeeded"
+    listed_id="$(printf '%s\n' "$comps" | grep -vE '^(Composition|----|$)' | awk '{print $1}' | grep -v '^$' | head -1)"
+    [ -z "$listed_id" ] && listed_id="$(printf '%s\n' "$comps" | awk 'NR==2{print $1}')"
+    if [ -n "$listed_id" ]; then
+      ok "found composition id: $listed_id"
+      [ -z "$comp_id" ] && comp_id="$listed_id"
+    else
+      fail "remotion compositions listed no usable composition id"
+    fi
+
+    # --- Mode 2: optional region test render (opt-in, writes a temp file) ---
+    if [ "$do_render" -eq 1 ] && [ -n "$comp_id" ]; then
+      if command -v ffprobe >/dev/null 2>&1; then
+        tmp_out="$(mktemp -t remotion-verify-XXXXXX).mp4"
+        printf '%s\n' "Rendering region test (frames 0-15) of '$comp_id' -> $tmp_out ..."
+        if (cd "$target" && npx --no-install remotion render "$comp_id" "$tmp_out" --frames=0-15 >/dev/null 2>&1) && [ -s "$tmp_out" ]; then
+          dims="$(ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=p=0 "$tmp_out" 2>/dev/null)"
+          if [ -n "$dims" ]; then
+            ok "render produced a non-empty MP4 with a video stream ($dims)"
+          else
+            fail "render produced a file but ffprobe found no video stream"
+          fi
+        else
+          fail "region test render failed or produced an empty file"
+        fi
+        rm -f "$tmp_out"
+      else
+        warn "--render requested but ffprobe not found; skipping the render check."
+      fi
+    elif [ "$do_render" -eq 0 ]; then
+      printf '%s\n' "(read-only mode: pass --render to also do a region test render + ffprobe)"
+    fi
+  else
+    warn "remotion CLI not runnable here (deps not installed?) — falling back to static checks."
+  fi
+fi
+
+# =============================================================================
+# Mode 3: static fallback (no node/CLI run, or CLI unavailable)
+# =============================================================================
+if [ "$ran_dynamic" -eq 0 ]; then
+  printf '%s\n' "Static checks (no runnable Remotion CLI):"
+  if [ -n "$comp_id" ]; then
+    ok "found a <Composition> with id: $comp_id"
+  else
+    fail "no <Composition id=\"...\"> found in any .tsx — a Remotion project needs at least one composition."
+  fi
+fi
+
+# --- render command references a real id + an output path (both modes) ------
+render_cmd="$(grep -rhoE 'remotion render[^\"'\''<]*' $tsx_files "$target"/*.json "$target"/*.ts "$target"/*.md "$target"/scripts/* 2>/dev/null | head -1)"
+if [ -n "$render_cmd" ]; then
+  if printf '%s' "$render_cmd" | grep -qE 'remotion render[[:space:]]+[A-Za-z0-9_-]+'; then
+    ok "a render command references a composition id"
+  else
+    warn "found 'remotion render' but no explicit composition id after it (a picker will prompt)."
+  fi
+  if printf '%s' "$render_cmd" | grep -qiE '\.(mp4|mov|webm|gif)'; then
+    ok "the render command names an output path"
+  else
+    warn "render command has no explicit output file (.mp4/.mov/.webm/.gif)."
+  fi
+else
+  warn "no 'remotion render' command found in the project files (scripts/, package.json, *.md)."
+fi
+
+printf '\n'
+if [ "$failures" -gt 0 ]; then
+  printf '%s\n' "${RED}$failures check(s) failed.${RESET}"
+  exit 1
+fi
+printf '%s\n' "${GREEN}All checks passed.${RESET}"
+exit 0