@lalalic/markcut 1.0.0

package/src/schema/index.ts

@@ -0,0 +1,259 @@
+import { z } from "zod";
+import { uid } from "../utils/index";
+
+/**
+ * Lite stream schema.
+ * Render-only — no prompts, no async transforms, no providers.
+ *
+ * A stream tree is rooted at `root`. Folders contain children.
+ * Leaf streams (video/audio/image/component) have actions[]
+ * describing when they appear on the timeline.
+ *
+ * Subtitles are a root-level overlay: root.subtitle points to a VTT file
+ * that covers the full video duration. Generated by the TTS pipeline.
+ */
+
+export const action = z.object({
+  id: z.string().default(() => uid()),
+  start: z.number().min(0).default(0).describe("seconds, relative to parent"),
+  end: z.number().min(0).default(1).describe("seconds, relative to parent"),
+  startFrom: z.number().optional().describe("trim seconds from source start"),
+  endAt: z.number().optional().describe("trim seconds at source end"),
+  loop: z.number().int().min(1).optional().describe(">1 = loop count"),
+  effectId: z.string().optional(),
+  style: z.string().optional().describe("inline css"),
+  volume: z.number().min(0).max(1).optional(),
+});
+export type Action = z.infer<typeof action>;
+
+export const eventSpec = z.object({
+  when: z.string().describe("'start' | 'end' | '50%' | '2.5s'"),
+  state: z.string().describe("JS expression evaluated with registered components in scope"),
+});
+export type EventSpec = z.infer<typeof eventSpec>;
+
+const BaseShape = {
+  id: z.string().default(() => uid()),
+  instruction: z.string().optional().describe("visual intent or style guide for this node (not rendered)"),
+  src: z.string().optional(),
+  style: z.string().optional().describe("inline css"),
+  visible: z.boolean().default(true),
+  isBackground: z.boolean().optional(),
+  durationInSeconds: z.number().optional().describe("set by engine; do not edit by hand"),
+  on: eventSpec.optional().describe("event that fires at a specific frame, mutating registered component state"),
+};
+
+export const base = z.object(BaseShape);
+
+export const folder = base.extend({
+  type: z.literal("folder").default("folder"),
+  isSeries: z.boolean().optional(),
+  transition: z.enum(["fade", "slide", "wipe", "flip", "clockWipe"]).optional(),
+  transitionTime: z.number().min(0.1).max(5).default(0.5),
+  shadow: z.number().min(0).optional(),
+  children: z.array(z.lazy((): z.ZodTypeAny => stream)).default(() => []),
+});
+export type Folder = z.infer<typeof folder>;
+
+export const subtitleOverlay = z.object({
+  src: z.string().describe("path or URL to VTT file covering the full video duration"),
+  style: z.string().optional().describe("inline css for the overlay container"),
+  fontSize: z.union([z.number(), z.string()]).optional(),
+  type: z.string().optional().describe("caption animation type: bounce, fade, typewriter, colorful, glowing, neon, etc."),
+  fontFamily: z.string().optional().describe("font family for subtitle text"),
+  fontStyle: z.string().optional().describe("font style: normal, italic, bold, bold italic, etc."),
+});
+export type SubtitleOverlay = z.infer<typeof subtitleOverlay>;
+
+export const root = folder.extend({
+  id: z.literal("root").default("root"),
+  type: z.literal("root").default("root"),
+  width: z.number().int().positive().default(1080),
+  height: z.number().int().positive().default(1920),
+  fps: z.number().int().positive().default(30),
+  instruction: z.string().optional(),
+  metadata: z.string().optional(),
+  stylesheet: z.string().optional().describe("global css; selectors use .type and .name"),
+  subtitle: subtitleOverlay.optional().describe("global subtitle overlay; src is a VTT file with absolute timestamps"),
+  /**
+   * Component registry source for react-jsx-parser.
+   *
+   * Two runtime forms (the descriptive ImportEntry[] array is resolved by the
+   * compiler/server BEFORE reaching here):
+   *   - string: URL to a pre-bundled ESM module whose named exports are the
+   *             components (set by the player server after bundling frontmatter
+   *             `imports:`).
+   *   - object: inline `{ Name: ComponentType }` map for programmatic use.
+   *
+   * Without this field declared, zod's default `.strip()` removes it during
+   * `root.parse()`, so the registry never loads at runtime.
+   */
+  imports: z
+    .union([z.string(), z.record(z.string(), z.any())])
+    .optional()
+    .describe("component registry: URL to a pre-bundled module (string) or inline component map (object)"),
+});
+export type Root = z.infer<typeof root>;
+
+export const video = base.extend({
+  type: z.literal("video").default("video"),
+  src: z.string().optional(),
+  volume: z.number().min(0).max(1).default(1),
+  playbackRate: z.number().optional(),
+  width: z.number().default(1080),
+  height: z.number().default(1920),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Video = z.infer<typeof video>;
+
+export const audio = base.extend({
+  type: z.literal("audio").default("audio"),
+  src: z.string().optional(),
+  volume: z.number().min(0).max(1).default(1),
+  foreground: z.boolean().optional().describe("ducks parent video audio while playing"),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Audio = z.infer<typeof audio>;
+
+export const image = base.extend({
+  type: z.literal("image").default("image"),
+  src: z.string().optional(),
+  fit: z.enum(["contain", "cover", "fill"]).default("contain"),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Image = z.infer<typeof image>;
+
+/**
+ * Component node — JSX usage expression compiled at runtime.
+ *
+ * The `jsx` field is a React JSX expression (e.g. "<BarChart data={...} />").
+ * Component tag names are resolved from the `components` map at runtime in compose context.
+
+ */
+export const component = base.extend({
+  type: z.literal("component").default("component"),
+  jsx: z.string().describe("usage JSX expression compiled at runtime; tag names resolved from imports"),
+  data: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional().describe("extra variables (e.g. from ~~~md source code fences) available in JSX scope"),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Component = z.infer<typeof component>;
+
+// ---------------------------------------------------------------------------
+// Effect — CSS keyframe animation wrapper
+// ---------------------------------------------------------------------------
+export const effect = base.extend({
+  type: z.literal("effect").default("effect"),
+  animation: z.string().optional().describe("builtin keyframe name or 'custom'"),
+  animationTimingFunction: z
+    .enum(["linear", "ease", "ease-in", "ease-out", "ease-in-out"])
+    .optional(),
+  animationIterationCount: z.number().default(1),
+  customKeyframes: z
+    .record(z.string(), z.record(z.string(), z.string()))
+    .optional()
+    .describe('inline keyframes: { "0": { opacity: "0" }, "100": { opacity: "1" } }'),
+  children: z.array(z.lazy((): z.ZodTypeAny => stream)).default(() => []),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Effect = z.infer<typeof effect>;
+
+// ---------------------------------------------------------------------------
+// Rhythm — audio loop with beat-synced children
+// ---------------------------------------------------------------------------
+export const rhythm = base.extend({
+  type: z.literal("rhythm").default("rhythm"),
+  src: z.string().optional().describe("audio file for beat playback"),
+  volume: z.number().min(0).max(1).default(1),
+  spots: z.array(z.number()).optional().describe("pre-computed beat timestamps in seconds"),
+  children: z.array(z.lazy((): z.ZodTypeAny => stream)).default(() => []),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type Rhythm = z.infer<typeof rhythm>;
+
+// ---------------------------------------------------------------------------
+// Include — references an external video JSON file by src.
+// The referenced file can be either:
+//   - A stream tree JSON (has `type: "root"` or `root` property)
+//   - A scene-based video.json (has `meta` and `scenes`)
+//
+// Usage:
+//   { type: "include", src: "./path/to/video.json", actions: [{ start: 0, end: 5 }] }
+//
+// Falls back to inline `children` if `src` is not set (legacy behavior).
+// ---------------------------------------------------------------------------
+export const include = base.extend({
+  type: z.literal("include").default("include"),
+  src: z.string().optional().describe("path or URL to video JSON file (stream tree or scene-based)"),
+  volume: z.number().min(0).max(1).default(1),
+  children: z.array(z.lazy((): z.ZodTypeAny => stream)).default(() => []),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+  imports: z.string().optional().describe("component bundle URL for sub-video (set by engine)"),
+});
+export type Include = z.infer<typeof include>;
+
+// ---------------------------------------------------------------------------
+// Scene — descriptive container node for storyboarding.
+// A scene is a visual grouping node with metadata (name, description) that
+// renders its children like a folder. It exists so UI tools can display
+// high-level storyboard cards without needing to understand the full stream
+// tree. The engine treats it identically to a folder.
+//
+// Usage:
+//   { type: "scene", name: "Intro", description: "Opening hook",
+//     children: [{ type: "component", componentName: "BigStatement", ... }] }
+//
+// Parent folder's isSeries controls sequencing (same as folder children).
+// ---------------------------------------------------------------------------
+export const scene = base.extend({
+  type: z.literal("scene").default("scene"),
+  name: z.string().optional().describe("scene name from heading"),
+  title: z.string().optional().describe("scene display title"),
+  instruction: z.string().optional().describe("scene visual intent for storyboard UI"),
+  children: z.array(z.lazy((): z.ZodTypeAny => stream)).default(() => []),
+  durationInSeconds: z.number().optional().describe("set by engine; do not edit by hand"),
+});
+export type Scene = z.infer<typeof scene>;
+
+// ---------------------------------------------------------------------------
+// Map — animated route visualization
+// ---------------------------------------------------------------------------
+export const mapWaypoint = z.object({
+  lat: z.number(),
+  lng: z.number(),
+  label: z.string().optional(),
+  media: z.string().optional().describe("image/video src for waypoint marker"),
+});
+
+export const mapStream = base.extend({
+  type: z.literal("map").default("map"),
+  waypoints: z.array(mapWaypoint).default(() => []),
+  routeColor: z.string().default("#4285F4"),
+  routeWeight: z.number().default(4),
+  zoom: z.number().default(10),
+  center: z.object({ lat: z.number(), lng: z.number() }).optional().describe("map view center (defaults to first waypoint)"),
+  mapType: z.enum(["roadmap", "satellite", "hybrid", "terrain"]).default("roadmap").describe("Google Maps style"),
+  travelMode: z.enum(["DRIVING", "WALKING", "BICYCLING", "TRANSIT"]).default("DRIVING").describe("Directions API travel mode"),
+  routeMarker: z.string().default("🚗").describe("emoji/character for the animated traveling marker"),
+  actions: z.array(action).min(1).default(() => [action.parse({})]),
+});
+export type MapStream = z.infer<typeof mapStream>;
+
+// ---------------------------------------------------------------------------
+// Discriminated union
+// ---------------------------------------------------------------------------
+export const stream = z.discriminatedUnion("type", [
+  root,
+  folder,
+  video,
+  audio,
+  image,
+  component,
+  effect,
+  rhythm,
+  mapStream,
+  include,
+  scene,
+]);
+export type Stream = z.infer<typeof stream>;
+
+

package/src/types/Audio.tsx

@@ -0,0 +1,56 @@
+import * as React from "react";
+import { Sequence, Audio as RemotionAudio, useRemotionEnvironment, useVideoConfig, staticFile } from "remotion";
+import { AudioContext, useFrameEvents } from "../context/index";
+import { toPlaybackRate } from "../utils/index";
+import type { Audio } from "../schema/index";
+
+function resolveAudioSrc(src: string): string {
+  if (/^(https?:|data:|blob:|file:|\/)/.test(src)) return src;
+  return staticFile(src);
+}
+
+export function AudioLeaf({ stream }: { stream: Audio }) {
+  const { fps } = useVideoConfig();
+  const environment = useRemotionEnvironment();
+  const ctx = React.useContext(AudioContext);
+  const totalDur = stream.durationInSeconds ?? stream.actions[0]?.end ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * fps)));
+  if (!stream.src) return null;
+  if (environment.isStudio) return null;
+
+  const resolvedSrc = resolveAudioSrc(stream.src);
+
+  return (
+    <>
+      {stream.actions.map((a) => {
+        const start = a.start ?? 0;
+        const end = a.end ?? start + 1;
+        const startFrom = a.startFrom ?? 0;
+        const endAt = a.endAt ?? stream.durationInSeconds ?? end - start;
+        const volume = a.volume ?? stream.volume ?? 1;
+        const playbackRate = a.loop ? 1 : toPlaybackRate((endAt - startFrom) / (end - start));
+        return (
+          <Sequence
+            key={a.id}
+            name={stream.src ?? "audio"}
+            durationInFrames={Math.max(1, Math.floor(fps * (end - start)))}
+            from={Math.floor(fps * start)}
+            layout="none"
+            showInTimeline={false}
+          >
+            <RemotionAudio
+              src={resolvedSrc}
+              startFrom={Math.floor(startFrom * fps)}
+              endAt={Math.floor(startFrom * fps) + Math.floor(((endAt - startFrom) * fps) / playbackRate)}
+              muted={volume === 0 || !!ctx?.foreground}
+              volume={volume}
+              loop={(a.loop ?? 1) > 1}
+              playbackRate={playbackRate}
+              showInTimeline={false}
+            />
+          </Sequence>
+        );
+      })}
+    </>
+  );
+}

package/src/types/Component.tsx

@@ -0,0 +1,135 @@
+import * as React from "react";
+import { Sequence, useVideoConfig } from "remotion";
+import { ComposeContext, useEventContext, useFrameEvents } from "../context/index";
+import JsxParser from "react-jsx-parser";
+import { useTweenBindings } from "../utils/tween";
+import type { Component, EventSpec } from "../schema/index";
+
+/**
+ * Wraps JsxParser with tween bindings for the current action.
+ * Must be a separate component so useTweenBindings follows Rules of Hooks.
+ */
+function TweenedJsxParser({
+  jsx,
+  components,
+  data,
+  action,
+}: {
+  jsx: string;
+  components: Record<string, React.ComponentType<any>> | undefined;
+  data: Record<string, unknown> | undefined;
+  action: { start?: number; end?: number };
+}) {
+  const tweenBindings = useTweenBindings(action);
+
+  return (
+    <>
+      <JsxParser
+        components={components as any}
+        bindings={React.useMemo(() => ({ ...data, ...tweenBindings }), [data, tweenBindings])}
+        jsx={jsx}
+        renderInWrapper={false}
+        blacklistedAttrs={[]}
+        disableKeyGeneration={true}
+        renderError={({ error }) => <div style={{ color: "red", padding: 20, fontSize: "larger" }}>{error}</div>}
+      />
+    </>
+  );
+}
+
+/**
+ * Component leaf — renders a JSX usage expression at runtime.
+ * Component tag names are resolved from ComposeContext.components
+ * (populated by the engine from root.imports).
+ *
+ * If the component has an `id`, it registers itself in EventContext so other
+ * nodes can mutate its state via `on` event specs.
+ */
+export function ComponentLeaf({ stream }: { stream: Component }) {
+  const { fps } = useVideoConfig();
+  const { components } = React.useContext(ComposeContext);
+  const eventCtx = useEventContext();
+
+  // Event-based state: other nodes can mutate this component's state
+  // via registered proxies in the event system.
+  const [eventState, setEventState] = React.useState<Record<string, any>>({});
+
+  // Use useLayoutEffect so registration completes BEFORE any useFrameEvents
+  // useEffect fires. Otherwise, at frame 0 the audio node's useFrameEvents
+  // evaluates "slide1.current=1" before this component has registered.
+  React.useLayoutEffect(() => {
+    if (stream.id) {
+      eventCtx.register(stream.id, (partial) =>
+        setEventState((prev) => ({ ...prev, ...partial })),
+      );
+      return () => eventCtx.unregister(stream.id);
+    }
+  }, [stream.id, eventCtx]);
+
+  // Merge: components registry → stream data (from code fences) → event state
+  const bindings = React.useMemo(
+    () => ({ ...components, ...stream.data, ...eventState }),
+    [stream.data, components, eventState],
+  );
+
+  if (!stream.jsx) return null;
+
+  return (
+    <>
+      {stream.actions.map((a) => {
+        const start = a.start ?? 0;
+        const end = a.end ?? start + 1;
+        const durFrames = Math.max(1, Math.floor(fps * (end - start)));
+        return (
+          <Sequence
+            key={a.id}
+            durationInFrames={durFrames}
+            from={Math.floor(fps * start)}
+            layout="none"
+          >
+            <EventAwareComponent
+              jsx={stream.jsx}
+              components={components}
+              data={bindings}
+              action={a}
+              durFrames={durFrames}
+              on={stream.on}
+            />
+          </Sequence>
+        );
+      })}
+    </>
+  );
+}
+
+/**
+ * Inner component that uses useFrameEvents (which requires useCurrentFrame).
+ * Must be separate so the hook is called inside Sequence context.
+ */
+function EventAwareComponent({
+  jsx,
+  components,
+  data,
+  action,
+  durFrames,
+  on,
+}: {
+  jsx: string;
+  components: Record<string, React.ComponentType<any>> | undefined;
+  data: Record<string, unknown> | undefined;
+  action: { start?: number; end?: number };
+  durFrames: number;
+  on?: EventSpec;
+}) {
+  // Fire events at the right frame for this node's timeline
+  useFrameEvents(on, durFrames);
+
+  return (
+    <TweenedJsxParser
+      jsx={jsx}
+      components={components}
+      data={data}
+      action={action}
+    />
+  );
+}

package/src/types/Effect.tsx

@@ -0,0 +1,88 @@
+/**
+ * Effect stream type — wraps children with keyframe-based CSS animation.
+ *
+ * Usage in stream tree:
+ *   { type: "effect", animation: "fadeIn", children: [{ type: "video", ... }] }
+ *
+ * The animation is computed frame-by-frame via Remotion's interpolate(),
+ * producing a wrapper <div> with the animated styles.
+ */
+import * as React from "react";
+import { useCurrentFrame, useVideoConfig } from "remotion";
+import { cssJS } from "../utils/index";
+import { useFrameEvents } from "../context/index";
+import { resolveAnimation, interpolateKeyframes } from "./keyframes";
+import type { Effect as EffectStream } from "../schema/index";
+
+export function EffectWrapper({
+  stream,
+  children,
+}: {
+  stream: EffectStream;
+  children: React.ReactNode;
+}) {
+  const frame = useCurrentFrame();
+  const { fps, width, height } = useVideoConfig();
+  const totalDur = stream.durationInSeconds ?? stream.actions[0]?.end ?? 1;
+  useFrameEvents(stream.on, Math.max(1, Math.floor(totalDur * fps)));
+
+  const actions = stream.actions ?? [];
+
+  const styles = React.useMemo(() => {
+    const result: Record<string, string>[] = [];
+
+    for (const action of actions) {
+      const start = Math.ceil(action.start * fps);
+      const end = Math.ceil(action.end * fps);
+      const durationInFrames = end - start;
+      if (durationInFrames <= 0) continue;
+
+      const animation = stream.animation;
+      const timingFn = stream.animationTimingFunction;
+      const iterCount = stream.animationIterationCount ?? 1;
+      const style = (cssJS(action.style) ?? {}) as Record<string, string>;
+
+      // Handle iteration count: loop the animation within the action range
+      let currentFrame = frame;
+      if (iterCount > 0 && durationInFrames > 0) {
+        const iteration = Math.floor((frame - start) / durationInFrames);
+        if (iteration < iterCount) {
+          currentFrame = start + ((frame - start) % durationInFrames);
+        }
+      }
+
+      if (currentFrame >= start && currentFrame < end) {
+        const actionFrame = currentFrame - start;
+
+        if (animation) {
+          const config = resolveAnimation(animation, stream.customKeyframes);
+          if (config) {
+            const animStyle = interpolateKeyframes(config, actionFrame, {
+              fps,
+              durationInSeconds: durationInFrames / fps,
+              timingFunction: timingFn,
+            });
+            if (animStyle) Object.assign(style, animStyle);
+          }
+        }
+
+        if (Object.keys(style).length > 0) {
+          result.push(style);
+        }
+      }
+    }
+
+    return result;
+  }, [frame, fps, actions, stream.animation, stream.animationTimingFunction, stream.animationIterationCount, stream.customKeyframes]);
+
+  if (styles.length === 0) return <>{children}</>;
+
+  return (
+    <div
+      style={Object.assign({ width, height, position: "absolute" as const, inset: 0 }, ...styles)}
+      className="effect"
+    >
+      {children}
+    </div>
+  );
+}