@lalalic/markcut 1.0.0

package/src/types/Map.tsx

@@ -0,0 +1,252 @@
+/**
+ * Map stream type — animated route on Google Maps.
+ *
+ * Renders a Google Map with Directions API route between waypoints and an
+ * animated marker that travels along the path in sync with the current frame.
+ * Uses @vis.gl/react-google-maps (Google Maps JS API wrapper) — no separate
+ * API key management needed beyond what's embedded in the engine build.
+ *
+ * Adapted from qili-ai studio's map component.
+ *
+ * Usage in stream tree:
+ *   {
+ *     type: "map",
+ *     waypoints: [{ lat, lng, label? }],
+ *     travelMode: "DRIVING",        // DRIVING | WALKING | BICYCLING
+ *     mapType: "roadmap",           // roadmap | satellite | hybrid | terrain
+ *     routeMarker: "🚗",            // emoji/char for animated pin
+ *     actions: [{ start: 0, end: 5 }]
+ *   }
+ */
+import React from "react";
+import { Sequence, useCurrentFrame, useVideoConfig, delayRender, continueRender } from "remotion";
+import { useFrameEvents } from "../context/index";
+import {
+  APIProvider, Map as GoogleMap, useMap, useMapsLibrary,
+  AdvancedMarker, Pin,
+} from "@vis.gl/react-google-maps";
+import type { MapStream } from "../schema/index";
+
+const GM_API_KEY = process.env.GOOGLE_MAPS_API_KEY || "";
+
+// ============================================================
+// MapLeaf — entry point, renders each action as a Sequence
+// ============================================================
+export function MapLeaf({ stream }: { stream: MapStream }) {
+  const { fps } = useVideoConfig();
+  const waypoints = stream.waypoints ?? [];
+  const totalDur = stream.durationInSeconds ?? stream.actions[0]?.end ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * fps)));
+  if (waypoints.length === 0) return null;
+
+  return (
+    <>
+      {stream.actions?.map((a, i) => {
+        const start = a.start ?? 0;
+        const end = a.end ?? start + 1;
+        const durFrames = Math.max(1, Math.floor(fps * (end - start)));
+        const center = stream.center ?? { lat: waypoints[0].lat, lng: waypoints[0].lng };
+        const zoom = stream.zoom ?? 10;
+        const mapType = stream.mapType ?? "roadmap";
+        const travelMode = stream.travelMode ?? "DRIVING";
+        const markerEmoji = stream.routeMarker ?? "🚗";
+        return (
+          <Sequence
+            key={a.id ?? i}
+            durationInFrames={durFrames}
+            from={Math.floor(fps * start)}
+            layout="none"
+          >
+            <APIProvider apiKey={GM_API_KEY}>
+              <GoogleMap
+                mapId={String(stream.id ?? i)}
+                defaultCenter={center}
+                defaultZoom={zoom}
+                defaultOptions={{
+                  mapTypeId: mapType,
+                  disableDefaultUI: true,
+                  zoomControl: false,
+                }}
+                style={{ width: "100%", height: "100%", position: "absolute" }}
+              >
+                <RouteWithMarker
+                  waypoints={waypoints}
+                  travelMode={travelMode}
+                  markerEmoji={markerEmoji}
+                  actionDuration={end - start}
+                />
+              </GoogleMap>
+            </APIProvider>
+          </Sequence>
+        );
+      })}
+    </>
+  );
+}
+
+// ============================================================
+// RouteWithMarker — gets the route via DirectionsService, then
+// renders an animated marker that follows the route path
+// ============================================================
+function RouteWithMarker({
+  waypoints, travelMode, markerEmoji, actionDuration,
+}: {
+  waypoints: { lat: number; lng: number; label?: string }[];
+  travelMode: string;
+  markerEmoji: string;
+  actionDuration: number;
+}) {
+  const map = useMap();
+  const routesLibrary = useMapsLibrary("routes");
+  const [leg, setLeg] = React.useState<google.maps.DirectionsLeg | null>(null);
+  const [routeIndex, setRouteIndex] = React.useState(0);
+  const handle = React.useRef<number | null>(null);
+
+  // Load directions
+  React.useEffect(() => {
+    if (!routesLibrary || !map || waypoints.length < 2) return;
+    const renderHandle = delayRender("Loading map directions...");
+    handle.current = renderHandle;
+
+    const renderer = new routesLibrary.DirectionsRenderer({ map, suppressMarkers: true });
+    const service = new routesLibrary.DirectionsService();
+
+    service
+      .route({
+        origin: waypoints[0],
+        destination: waypoints[waypoints.length - 1],
+        waypoints: waypoints.slice(1, -1).map((wp) => ({ location: wp, stopover: true })),
+        travelMode: google.maps.TravelMode[travelMode as keyof typeof google.maps.TravelMode],
+        provideRouteAlternatives: false,
+      })
+      .then((response) => {
+        renderer.setDirections(response);
+        setRouteIndex(0);
+        setLeg(response.routes[0]?.legs[0] ?? null);
+        if (handle.current !== null) continueRender(handle.current);
+      })
+      .catch(() => {
+        if (handle.current !== null) continueRender(handle.current);
+      });
+
+    return () => {
+      renderer.setMap(null);
+    };
+  }, [routesLibrary, map, waypoints, travelMode]);
+
+  // Update route index
+  React.useEffect(() => {
+    setRouteIndex((prev) => prev);
+  }, [routeIndex]);
+
+  // Compute animated marker position
+  const position = useAnimatedPosition({ leg, actionDuration, waypoints });
+
+  return (
+    <>
+      {waypoints.map((wp, i) => (
+        <AdvancedMarker key={i} position={wp}>
+          {wp.label ? (
+            <div
+              style={{
+                background: "rgba(255,255,255,0.9)",
+                borderRadius: "4px",
+                padding: "2px 6px",
+                fontSize: "12px",
+                fontWeight: 700,
+                color: "#333",
+                whiteSpace: "nowrap",
+                position: "relative",
+                top: "-24px",
+              }}
+            >
+              {wp.label}
+            </div>
+          ) : null}
+        </AdvancedMarker>
+      ))}
+      {position ? (
+        <AdvancedMarker position={position}>
+          <Pin glyphText={markerEmoji} scale={4} />
+        </AdvancedMarker>
+      ) : null}
+    </>
+  );
+}
+
+// ============================================================
+// useAnimatedPosition — returns the current lat/lng of the
+// animated marker along the route path
+// ============================================================
+function useAnimatedPosition({
+  leg, actionDuration, waypoints,
+}: {
+  leg: google.maps.DirectionsLeg | null;
+  actionDuration: number;
+  waypoints: { lat: number; lng: number }[];
+}) {
+  const frame = useCurrentFrame();
+  const { fps } = useVideoConfig();
+
+  return React.useMemo(() => {
+    if (!leg || !leg.duration?.value) {
+      // Fallback: linear interpolation between waypoints
+      if (waypoints.length < 2) return null;
+      const t = Math.min(frame / (actionDuration * fps), 1);
+      const total = waypoints.length - 1;
+      const segI = Math.min(Math.floor(t * total), total - 1);
+      const segT = (t * total) - segI;
+      const a = waypoints[segI];
+      const b = waypoints[segI + 1];
+      if (!a || !b) return null;
+      return {
+        lat: a.lat + (b.lat - a.lat) * segT,
+        lng: a.lng + (b.lng - a.lng) * segT,
+      };
+    }
+
+    // Follow the route path using leg steps
+    const currentInSecond = (frame / fps) * (leg.duration.value / actionDuration);
+    const { step, elapsedInSeconds } = getCurrentStep(leg, currentInSecond);
+    if (!step || !step.path) {
+      // Fallback to linear
+      const t = Math.min(frame / (actionDuration * fps), 1);
+      const total = waypoints.length - 1;
+      const segI = Math.min(Math.floor(t * total), total - 1);
+      const segT = (t * total) - segI;
+      const a = waypoints[segI];
+      const b = waypoints[segI + 1];
+      if (!a || !b) return null;
+      return {
+        lat: a.lat + (b.lat - a.lat) * segT,
+        lng: a.lng + (b.lng - a.lng) * segT,
+      };
+    }
+
+    const stepElapsed = currentInSecond - elapsedInSeconds;
+    const stepProgress = stepElapsed / (step.duration?.value ?? 1);
+    const pathIdx = Math.min(
+      Math.max(0, Math.floor(stepProgress * step.path.length)),
+      step.path.length - 1,
+    );
+    const pt = step.path[pathIdx];
+    if (!pt) return null;
+    return { lat: pt.lat(), lng: pt.lng() };
+  }, [leg, frame, fps, actionDuration, waypoints]);
+}
+
+// ============================================================
+// getCurrentStep — finds which step of a DirectionsLeg the
+// current time falls into
+// ============================================================
+function getCurrentStep(leg: google.maps.DirectionsLeg, currentInSecond: number) {
+  let elapsedInSeconds = 0;
+  for (const step of leg.steps ?? []) {
+    const stepDur = step.duration?.value ?? 0;
+    if (elapsedInSeconds <= currentInSecond && currentInSecond < elapsedInSeconds + stepDur) {
+      return { step, elapsedInSeconds };
+    }
+    elapsedInSeconds += stepDur;
+  }
+  return { step: null, elapsedInSeconds: 0 };
+}

package/src/types/Rhythm.tsx

@@ -0,0 +1,58 @@
+/**
+ * Rhythm stream type — audio playback with loop + beat-synced children.
+ *
+ * The rhythm stream plays audio in a loop. Children are expected to have
+ * their timing pre-computed (by a build step or CLI tool) to align with
+ * detected beat positions stored in `spots[]`.
+ *
+ * Usage in stream tree:
+ *   { type: "rhythm", src: "beat.mp3", spots: [0.5, 1.2, 1.9], children: [...] }
+ */
+import * as React from "react";
+import { Sequence, Audio as RemotionAudio, useRemotionEnvironment, useVideoConfig, staticFile } from "remotion";
+import { useFrameEvents } from "../context/index";
+import type { Rhythm } from "../schema/index";
+
+function resolveAudioSrc(src: string): string {
+  if (/^(https?:|data:|blob:|file:|\/)/.test(src)) return src;
+  return staticFile(src);
+}
+
+export function RhythmLeaf({ stream }: { stream: Rhythm }) {
+  const { fps } = useVideoConfig();
+  const environment = useRemotionEnvironment();
+  const totalDur = stream.durationInSeconds ?? stream.actions[0]?.end ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * fps)));
+
+  if (!stream.src || environment.isStudio) return null;
+
+  const resolvedSrc = resolveAudioSrc(stream.src);
+
+  return (
+    <>
+      {stream.actions.map((a) => {
+        const start = a.start ?? 0;
+        const end = a.end ?? start + 1;
+        const volume = a.volume ?? stream.volume ?? 1;
+        return (
+          <Sequence
+            key={a.id}
+            name={stream.src ?? "rhythm"}
+            durationInFrames={Math.max(1, Math.floor(fps * (end - start)))}
+            from={Math.floor(fps * start)}
+            layout="none"
+            showInTimeline={false}
+          >
+            <RemotionAudio
+              src={resolvedSrc}
+              muted={volume === 0}
+              volume={volume}
+              loop
+              showInTimeline={false}
+            />
+          </Sequence>
+        );
+      })}
+    </>
+  );
+}

package/src/types/Scene.tsx

@@ -0,0 +1,42 @@
+/**
+ * SceneLeaf — renders a descriptive container node for storyboarding.
+ *
+ * A scene is purely a visual grouping node with metadata (name, description)
+ * that renders its children like a folder. It exists so UI tools can display
+ * high-level storyboard cards without needing to parse the full stream tree.
+ *
+ * The engine treats it identically to a folder at render time.
+ */
+import * as React from "react";
+import { Folder } from "remotion"
+import { ComposeContext, AudioContext, useFrameEvents } from "../context/index";
+import { cssJS, toClassName } from "../utils/index";
+import type { Scene as SceneStream } from "../schema/index";
+import { FolderLeaf } from "./Folder";
+
+export function SceneLeaf({ stream }: { stream: SceneStream }) {
+  const { Container } = React.useContext(ComposeContext);
+  const parentAudio = React.useContext(AudioContext);
+  const totalDur = stream.durationInSeconds ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * 30)));
+
+  const audioCtx = React.useMemo(
+    () => ({ id: stream.id, parent: parentAudio }),
+    [stream.id, parentAudio],
+  );
+
+  return (
+    <AudioContext.Provider value={audioCtx as any}>
+      <Container
+        id={stream.id}
+        type="scene"
+        style={cssJS(stream.style) as React.CSSProperties}
+        className={`scene ${toClassName(stream.name ?? "")}`}
+      >
+        <Folder name={stream.name ?? ""}>
+          <FolderLeaf stream={stream as any} />
+        </Folder>
+      </Container>
+    </AudioContext.Provider>
+  );
+}

package/src/types/Subtitle.tsx

@@ -0,0 +1,218 @@
+import * as React from "react";
+import { cancelRender, continueRender, delayRender, Sequence, staticFile, useVideoConfig } from "remotion";
+import * as Subtitles from "remotion-subtitle";
+import { cssJS, parseVTT, type Cue } from "../utils/index";
+import type { SubtitleOverlay as SubtitleOverlayConfig } from "../schema/index";
+
+const { SubtitleSequence: _SubtitleSeq, ...CaptionComponents } = Subtitles;
+
+function resolveSubtitleSrc(src: string): string {
+  if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:") || src.startsWith("/")) {
+    return src;
+  }
+  return staticFile(src);
+}
+
+const DEFAULT_BOX_STYLE: React.CSSProperties = {
+  position: "absolute",
+  inset: 0,
+  display: "flex",
+  alignItems: "flex-end",
+  justifyContent: "center",
+  padding: "0 5% 8% 5%",
+  pointerEvents: "none",
+  zIndex: 100,
+};
+
+const DEFAULT_TEXT_STYLE: React.CSSProperties = {
+  color: "white",
+  fontSize: 56,
+  fontWeight: 700,
+  textAlign: "center",
+  textShadow: "0 2px 12px rgba(0,0,0,0.85)",
+  lineHeight: 1.2,
+  fontFamily: '"PingFang SC","Noto Sans CJK SC","Hiragino Sans","Helvetica Neue",sans-serif',
+};
+
+/**
+ * Resolve the caption component from a type string.
+ * "typewriter" → TypewriterCaption, "fade" → FadeCaption, etc.
+ * Falls back to base Caption component when type is omitted or unknown.
+ */
+function resolveCaption(type?: string): React.FC<{ text: any; style?: React.CSSProperties }> {
+  if (!type) return CaptionComponents.Caption;
+  const key = type.charAt(0).toUpperCase() + type.slice(1) + "Caption" as keyof typeof CaptionComponents;
+  return (CaptionComponents[key] ?? CaptionComponents.Caption) as React.FC<{ text: any; style?: React.CSSProperties }>;
+}
+
+/**
+ * Wraps text so that it renders HTML via dangerouslySetInnerHTML while
+ * still being compatible with caption components that call .length,
+ * .slice(), or .split() on the text prop (e.g. TypewriterCaption).
+ *
+ * The returned object is a React element (span with innerHTML) augmented
+ * with string-like methods so animation components can iterate over
+ * characters while respecting HTML tag boundaries.
+ */
+function supportHtml(text: string): any {
+  const el = { ...<span dangerouslySetInnerHTML={{ __html: text }} /> };
+  Object.assign(el, {
+    length: text.length,
+    slice(start: number, end: number) {
+      const chars: string[] = [];
+      let inTag = false;
+      let counter = 0;
+      for (const char of text) {
+        if (inTag) {
+          chars.push(char);
+          if (char === ">") inTag = false;
+          continue;
+        }
+        if (char === "<") {
+          chars.push(char);
+          inTag = true;
+        } else if (char === ">") {
+          chars.push(char);
+          inTag = false;
+        } else {
+          if (counter <= end) chars.push(char);
+          counter++;
+        }
+      }
+      return <span dangerouslySetInnerHTML={{ __html: chars.join("") }} />;
+    },
+    split(separator: string | RegExp, limit?: number) {
+      const stripped = text.replace(/<.*?>/g, "");
+      return stripped.split(separator, limit);
+    },
+  });
+  return el;
+}
+
+/**
+ * Renders a single cue inside a <Sequence> so Remotion only evaluates it
+ * during the cue's active time window — matching the qili-ai pattern.
+ * This avoids per-frame cue lookups and lets the caption component mount
+ * once per cue rather than re-creating on every frame.
+ */
+function CueFrame({
+  cue,
+  fps,
+  CaptionComponent,
+  subtitle,
+}: {
+  cue: Cue;
+  fps: number;
+  CaptionComponent: React.FC<{ text: any; style?: React.CSSProperties }>;
+  subtitle: SubtitleOverlayConfig;
+}) {
+  const durationInFrames = Math.max(1, Math.floor((cue.endAt - cue.startFrom) * fps));
+  const from = Math.floor(cue.startFrom * fps);
+
+  // Stable reference: compute once per cue, not every frame
+  const captionText = React.useMemo(() => supportHtml(cue.text), [cue.text]);
+  const textStyle: React.CSSProperties = React.useMemo(
+    () => ({
+      ...DEFAULT_TEXT_STYLE,
+      fontSize: subtitle.fontSize ?? DEFAULT_TEXT_STYLE.fontSize,
+      fontFamily: subtitle.fontFamily ?? DEFAULT_TEXT_STYLE.fontFamily,
+      fontStyle: subtitle.fontStyle as any,
+    }),
+    [subtitle.fontSize, subtitle.fontFamily, subtitle.fontStyle],
+  );
+
+  return (
+    <Sequence layout="none" durationInFrames={durationInFrames} from={from}>
+      <CaptionComponent text={captionText} style={textStyle} />
+    </Sequence>
+  );
+}
+
+/**
+ * Root-level subtitle overlay. Renders a single VTT file as captions over
+ * the entire video. VTT cue timestamps are absolute (seconds from video start),
+ * matching useCurrentFrame() / fps directly.
+ *
+ * Each cue is rendered as a separate <Sequence> (qili-ai pattern) so Remotion
+ * only mounts/evaluates the caption component during its active time window.
+ * This avoids per-frame computation for inactive cues.
+ *
+ * Supports a `type` field to select the caption animation component:
+ *   "bounce", "fade", "typewriter", "colorful", "glowing", "neon", etc.
+ * Defaults to plain `Caption` when type is omitted.
+ *
+ * Mounted once inside AbsoluteFill in MarkCut when root.subtitle is set.
+ */
+export function SubtitleOverlay({ subtitle }: { subtitle: SubtitleOverlayConfig }) {
+  const { fps } = useVideoConfig();
+  const [cues, setCues] = React.useState<Cue[] | null>(null);
+
+  const CaptionComponent = React.useMemo(
+    () => resolveCaption(subtitle.type),
+    [subtitle.type],
+  );
+
+  React.useEffect(() => {
+    const { src } = subtitle;
+
+    // Inline VTT body
+    if (src.includes("-->")) {
+      setCues(parseVTT(src));
+      return;
+    }
+
+    if (!/\.vtt(?:$|[?#])/.test(src)) {
+      // Plain text — single static caption for the whole video
+      setCues([{ startFrom: 0, endAt: Infinity, text: src }]);
+      return;
+    }
+
+    const handle = delayRender(`Loading subtitles: ${src}`);
+    let active = true;
+    let settled = false;
+    const finish = () => {
+      if (settled) return;
+      settled = true;
+      continueRender(handle);
+    };
+
+    fetch(resolveSubtitleSrc(src))
+      .then((r) => {
+        if (!r.ok) throw new Error(`Failed to load subtitles (${r.status}): ${src}`);
+        return r.text();
+      })
+      .then((text) => {
+        if (!active) return;
+        setCues(parseVTT(text));
+        finish();
+      })
+      .catch((err) => {
+        if (!active) return;
+        settled = true;
+        cancelRender(err);
+      });
+
+    return () => {
+      active = false;
+      finish();
+    };
+  }, [subtitle.src]);
+
+  if (!cues) return null;
+
+  const boxCss = subtitle.style ? (cssJS(subtitle.style) as React.CSSProperties) : {};
+
+  return (
+    <div className={`${subtitle.type || "default"} subtitle-overlay`} style={{ ...DEFAULT_BOX_STYLE, ...boxCss }}>
+      {cues.map((cue, i) => (
+        <CueFrame
+          key={`${i}-${cue.startFrom}-${cue.endAt}`}
+          cue={cue}
+          fps={fps}
+          CaptionComponent={CaptionComponent}
+          subtitle={subtitle}
+        />
+      ))}
+    </div>
+  );
+}

package/src/types/Video.tsx

@@ -0,0 +1,70 @@
+import * as React from "react";
+import { Sequence, OffthreadVideo, useVideoConfig, staticFile } from "remotion";
+
+function resolveVideoSrc(src: string): string {
+  if (/^(https?:|data:|blob:|file:|\/)/.test(src)) return src;
+  return staticFile(src);
+}
+import { ComposeContext, AudioContext, useFrameEvents } from "../context/index";
+import { toPlaybackRate, cssJS } from "../utils/index";
+import type { Video } from "../schema/index";
+import { FrameSyncStyle } from "./FrameSyncStyle";
+
+export function VideoLeaf({ stream }: { stream: Video }) {
+  const { fps } = useVideoConfig();
+  const audio = React.useContext(AudioContext);
+  const totalDur = stream.durationInSeconds ?? stream.actions[0]?.end ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * fps)));
+  if (!stream.src) return null;
+  const resolvedSrc = resolveVideoSrc(stream.src);
+
+  return (
+    <>
+      {stream.actions.map((a) => {
+        const start = a.start ?? 0;
+        const end = a.end ?? start + 1;
+        const startFrom = a.startFrom ?? 0;
+        const endAt = a.endAt ?? stream.durationInSeconds ?? end - start;
+        const volume = a.volume ?? stream.volume ?? 1;
+        const playbackRate = a.loop ? 1 : Math.min(1, toPlaybackRate((endAt - startFrom) / (end - start)));
+        const actionStyle = cssJS(a.style);
+        const hasAnimation = "animation" in actionStyle;
+        return (
+          <Sequence
+            key={a.id}
+            durationInFrames={Math.max(1, Math.floor(fps * (end - start)))}
+            from={Math.floor(fps * start)}
+            layout="none"
+            showInTimeline={false}
+          >
+            {hasAnimation ? (
+              <FrameSyncStyle style={actionStyle}>
+                <OffthreadVideo
+                  src={resolvedSrc}
+                  startFrom={Math.floor(startFrom * fps)}
+                  endAt={Math.floor(startFrom * fps) + Math.floor(((endAt - startFrom) * fps) / playbackRate)}
+                  muted={volume === 0 || !!audio?.foreground}
+                  volume={volume}
+                  playbackRate={playbackRate}
+                  showInTimeline={false}
+                  style={{ width: "100%", height: "100%" }}
+                />
+              </FrameSyncStyle>
+            ) : (
+              <OffthreadVideo
+                src={resolvedSrc}
+                startFrom={Math.floor(startFrom * fps)}
+                endAt={Math.floor(startFrom * fps) + Math.floor(((endAt - startFrom) * fps) / playbackRate)}
+                muted={volume === 0 || !!audio?.foreground}
+                volume={volume}
+                playbackRate={playbackRate}
+                showInTimeline={false}
+                style={{ width: "100%", height: "100%", ...actionStyle }}
+              />
+            )}
+          </Sequence>
+        );
+      })}
+    </>
+  );
+}