@lalalic/markcut 1.1.1 → 2.0.0

package/src/descriptive/resolve.ts

@@ -261,6 +261,117 @@ export async function resolveMediaDurations(
   return clone;
 }
 
+// ── Dialogue expansion ──────────────────────────────────────────────────────
+
+/**
+ * Detect if a script contains multi-turn dialogue in `SpeakerName: text` format.
+ *
+ * A script is a "dialogue" if at least 2 lines match the pattern `Name: text`.
+ * Single-line `SpeakerName: text` is NOT treated as dialogue — it's just a
+ * regular script with a speaker annotation.
+ *
+ * Example dialogue:
+ * ```
+ * Ray: Hello everyone and welcome
+ * Alice: Good day to you all
+ * Ray: Let's get started
+ * ```
+ *
+ * Each line becomes a separate audio node with:
+ * - `script`: just the text (without the "SpeakerName: " prefix)
+ * - `speaker`: the speaker name
+ *
+ * The dialogue lines are wrapped in a series container so they play sequentially
+ * regardless of the parent scene's layout.
+ */
+export function parseDialogueLines(script: string): Array<{ speaker: string; text: string }> {
+  const lines = script.split("\n").map(l => l.trim()).filter(Boolean);
+  const dialoguePattern = /^([\w\u4e00-\u9fff][\w\s\u4e00-\u9fff]*?):\s*(.+)$/;
+
+  const result: Array<{ speaker: string; text: string }> = [];
+  for (const line of lines) {
+    const match = line.match(dialoguePattern);
+    if (!match) return []; // Any non-matching line invalidates the dialogue format
+    result.push({ speaker: match[1]!.trim(), text: match[2]!.trim() });
+  }
+
+  // Require at least 2 lines to be considered a dialogue
+  return result.length >= 2 ? result : [];
+}
+
+/**
+ * Walk the descriptive tree and expand multi-turn dialogue-formatted scripts
+ * into sequential per-line audio nodes.
+ *
+ * This runs BEFORE resolveScripts so each dialogue line gets its own TTS
+ * generation with the correct per-speaker voice.
+ */
+export function resolveDialogue(root: DescriptiveRoot): DescriptiveRoot {
+  const clone: DescriptiveRoot = JSON.parse(JSON.stringify(root));
+
+  // Collect dialogue nodes for expansion (walking and modifying simultaneously is tricky)
+  const expansions: Array<{
+    parent: any;
+    index: number;
+    originalNode: any;
+    lines: Array<{ speaker: string; text: string }>;
+  }> = [];
+
+  walkDown(clone as any, (node, parent) => {
+    if (node.type !== "audio") return;
+    if (!node.script || typeof node.script !== "string") return;
+    if (node.src) return; // Already has a real source
+
+    const lines = parseDialogueLines(node.script);
+    if (lines.length < 2) return; // Not a dialogue
+
+    if (!parent || !Array.isArray(parent.children)) return;
+    const idx = parent.children.indexOf(node);
+    if (idx === -1) return;
+
+    expansions.push({ parent, index: idx, originalNode: node, lines });
+  });
+
+  if (expansions.length === 0) return root;
+
+  // Apply expansions in reverse index order so splice indices stay valid
+  expansions.sort((a, b) => b.index - a.index);
+
+  for (const { parent, index, originalNode, lines } of expansions) {
+    const dialogueNodes: any[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i]!;
+      const { speaker, text } = line;
+      const lineNode: any = {
+        ...originalNode,
+        id: originalNode.id ? `${originalNode.id}-line-${i}` : undefined,
+        script: text,
+        speaker,
+        start: 0,
+        duration: undefined, // Will be set by TTS duration probing
+      };
+      delete lineNode.src;
+      dialogueNodes.push(lineNode);
+    }
+
+    // Wrap dialogue lines in a series container for sequential playback
+    const seriesContainer: any = {
+      type: "series",
+      id: `${originalNode.id ?? "dialogue"}-series`,
+      children: dialogueNodes,
+    };
+
+    // Replace the original node with the series container
+    parent.children.splice(index, 1, seriesContainer);
+  }
+
+  const expanded = expansions.length > 0 ? clone : root;
+  const count = expansions.reduce((sum, e) => sum + e.lines.length, 0);
+  console.log(`  💬 dialogue: expanded ${expansions.length} script${expansions.length > 1 ? "s" : ""} into ${count} lines`);
+  return expanded;
+}
+
 // ── Script → TTS → STT ─────────────────────────────────────────────────────
 
 export interface ResolveScriptOptions {
@@ -292,23 +403,30 @@ export async function resolveScripts(
   let cacheDirty = false;
 
   // Collect all audio nodes that have script text but no src yet
-  // Also store their parent scene for per-scene tts resolution
-  const allScriptNodes: Array<{ node: any; id: string; parent: any }> = [];
-  walkDown(clone as any, (node, parent) => {
+  const allScriptNodes: Array<{ node: any; id: string }> = [];
+  walkDown(clone as any, (node) => {
     if (node.type !== "audio") return;
     if (!node.script || typeof node.script !== "string") return;
     if (node.src) return; // already has real source
     const id = node.id ?? `audio-${allScriptNodes.length}`;
-    allScriptNodes.push({ node, id, parent });
+    allScriptNodes.push({ node, id });
   });
 
   if (allScriptNodes.length > 0) {
     console.log(`  🔊 TTS: generating ${allScriptNodes.length} script${allScriptNodes.length > 1 ? "s" : ""}...`);
   }
 
-  for (const { node, id, parent } of allScriptNodes) {
-    // Resolve TTS config: parent scene tts > root tts > options ttsCli > default
-    const ttsCli = parent?.tts ?? clone.tts ?? options.ttsCli ?? DEFAULT_TTS_CLI;
+  for (const { node, id } of allScriptNodes) {
+    // TTS CLI from root config only
+    let ttsCli = clone.tts ?? options.ttsCli ?? DEFAULT_TTS_CLI;
+
+    // Per-speaker voice appends extra CLI flags from root voices config
+    if (node.speaker && clone.voices) {
+      const speakerVoice = clone.voices[node.speaker];
+      if (speakerVoice) {
+        ttsCli = `${ttsCli} ${speakerVoice}`;
+      }
+    }
 
     // Content-addressed filename: hash of script + CLI, so identical scripts
     // across different source files share the same audio file.
@@ -318,7 +436,8 @@ export async function resolveScripts(
     // Check cache — skip TTS if script + config unchanged AND audio file exists
     const cached = checkCache(cache, `tts:${cacheKey}`, cacheKey);
     let generated: string;
-    const label = firstWords(node.script, 8);
+    const speakerLabel = node.speaker ? `${node.speaker}: ` : "";
+    const label = `${speakerLabel}${firstWords(node.script, 8)}`;
     if (cached) {
       generated = cached;
       console.log(`  ✓ TTS: ${label} (cached)`);
@@ -378,15 +497,16 @@ export async function resolveSubtitles(
   let cacheDirty = false;
 
   // Collect { audioSrc, absoluteOffset } from the compiled tree
-  const clips: Array<{ audioSrc: string; offset: number }> = [];
+  const clips: Array<{ audioSrc: string; offset: number; speaker?: string }> = [];
 
   /**
    * Walk an array of sibling nodes, tracking cumulative offset for series layouts.
    *
-   * In the compiled stream tree, container nodes (Folder/Scene) don't carry `actions`
-   * — their position on the timeline is implicit in the `<Series>`/`<TransitionSeries>`
-   * renderer which plays each child sequentially based on `durationInSeconds`.
-   * This function replicates that logic to compute absolute audio offsets.
+   * In the compiled stream tree, container nodes (Folder/Scene) don't carry leaf
+   * timing (start/end) — their position on the timeline is implicit in the
+   * `<Series>`/`<TransitionSeries>` renderer which plays each child sequentially
+   * based on `durationInSeconds`. This function replicates that logic to compute
+   * absolute audio offsets.
    */
   function walkSiblings(
     nodes: any[],
@@ -404,13 +524,13 @@ export async function resolveSubtitles(
       // In a series, each child starts after all previous siblings' durations.
       // In parallel, all children share the parent offset.
       const nodeStart = parentIsSeries ? seriesOffset : parentOffset;
-      // Leaf nodes (video/audio/image/component) carry an action start offset
-      // relative to the container start (e.g., parallel layout with staggered start).
-      const actionStart = node.actions?.[0]?.start ?? 0;
+      // Leaf nodes carry a start offset relative to the container start
+      // (e.g., parallel layout with staggered start) directly on the base field.
+      const actionStart = node.start ?? 0;
       const effectiveOffset = nodeStart + actionStart;
 
       if (node.type === "audio" && node.src) {
-        clips.push({ audioSrc: node.src, offset: effectiveOffset });
+        clips.push({ audioSrc: node.src, offset: effectiveOffset, speaker: node.speaker });
       }
 
       // Recurse into children — determine their layout from the node type
@@ -449,7 +569,7 @@ export async function resolveSubtitles(
     }
   }
 
-  // Walk the compiled tree (with actions) for correct absolute offsets;
+  // Walk the compiled tree (with base timing) for correct absolute offsets;
   // fall back to the descriptive tree if no compiled tree provided.
   const treeToWalk = options.compiled ?? (clone as any);
   walkSiblings(
@@ -466,7 +586,7 @@ export async function resolveSubtitles(
 
   let sttFailedCount = 0;
 
-  for (const { audioSrc, offset } of clips) {
+  for (const { audioSrc, offset, speaker } of clips) {
     // Cache key: audio hash + STT CLI string
     const audioHash = existsSync(audioSrc)
       ? createHash("sha1").update(readFileSync(audioSrc)).digest("hex").slice(0, 12)
@@ -518,7 +638,11 @@ export async function resolveSubtitles(
         const sec = (s % 60).toFixed(3).padStart(6, "0");
         return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${sec}`;
       };
-      const text = lines.slice(lines.indexOf(tline) + 1).join("\n").trim();
+      let text = lines.slice(lines.indexOf(tline) + 1).join("\n").trim();
+      // Prepend speaker prefix for dialogue cues
+      if (speaker) {
+        text = `${speaker}: ${text}`;
+      }
       mergedLines.push(String(cueIndex++));
       mergedLines.push(`${formatSec(toSec(a) + offset)} --> ${formatSec(toSec(z) + offset)}`);
       mergedLines.push(text, "");
@@ -864,6 +988,10 @@ export async function resolveAll(
     skip: options.skip,
   });
 
+  // Step 4.5: Expand multi-turn dialogue (SpeakerName: text format)
+  // Must run before TTS so each dialogue line gets its own audio node
+  result = resolveDialogue(result);
+
   // Step 5: Script → TTS
   if (options.scriptOutputDir) {
     result = await resolveScripts(result, {

package/src/entry.tsx

@@ -1,5 +1,6 @@
 import * as React from "react";
-import { AbsoluteFill, continueRender, delayRender } from "remotion";
+import { AbsoluteFill, continueRender, delayRender, staticFile } from "remotion";
+import { ensureSharedImportMap } from "./utils/component-import-map";
 import { ComposeContext, EventProvider, type ComposeContextValue } from "./context/index";
 
 import { FolderLeaf } from "./types/Folder";
@@ -58,7 +59,12 @@ function useComponentRegistry(imports: unknown): Record<string, React.ComponentT
       if (!handleRef.current) {
         handleRef.current = delayRender("Loading component registry: " + imports);
       }
-      import(/* webpackIgnore: true */ imports)
+      // Absolute URLs and "/"-rooted paths (preview server) load as-is;
+      // relative paths (render CLI stages bundles into publicDir) resolve
+      // through staticFile so the Remotion render server can serve them.
+      const moduleUrl = /^(https?:|data:|blob:|\/)/.test(imports) ? imports : staticFile(imports);
+      ensureSharedImportMap();
+      import(/* webpackIgnore: true */ moduleUrl)
         .then((mod: any) => {
           // The bundle exports all components as named exports
           setRegistry(mod.default ?? mod);

package/src/player/browser.tsx

@@ -2,6 +2,11 @@
  * Browser player entry point.
  * Bundled with esbuild and served by the player server.
  * Renders stream tree JSON using @remotion/player with MarkCut.
+ *
+ * Supports three modes passed via window.MODE:
+ *   "label"  — label annotation overlay
+ *   "edit"   — edit input with AI auto-reload
+ *   (default) — plain preview with optional variant bar
  */
 import * as React from "react";
 import { createRoot } from "react-dom/client";
@@ -9,6 +14,7 @@ import * as ReactDOM from "react-dom";
 import * as Remotion from "remotion";
 import { Player } from "@remotion/player";
 import { MarkCut, getDurationInSeconds } from "../entry";
+import { HeaderBar, EditControls, LabelControls, SceneThumbnails, VariantBar } from "./components/index";
 
 /**
  * Register all player-bundled packages on a global registry so the import map
@@ -106,25 +112,58 @@ function PlayerApp() {
   const [ready, setReady] = React.useState(false);
   const [error, setError] = React.useState<string | null>(null);
   const [data, setData] = React.useState<any>(null);
-  const [refreshKey, setRefreshKey] = React.useState(0);
   const [muted, setMuted] = React.useState(false);
   const [volume, setVolume] = React.useState(1);
-  const seekAttemptedRef = React.useRef(false);
   const mountedRef = React.useRef(true);
 
+  // ── Track current player frame without re-renders ───────────────────
+  const currentFrameRef = React.useRef(0);
+  const [currentTime, setCurrentTime] = React.useState(0);
+  const [activeScene, setActiveScene] = React.useState("");
+
+  // Fetch scenes for active scene tracking
+  React.useEffect(() => {
+    fetch("/api/video-info")
+      .then((r) => r.json())
+      .then((info) => {
+        if (info.scenes) {
+          // Store scenes globally for time-based lookup
+          (window as any).__scenes = info.scenes;
+        }
+      })
+      .catch(() => {});
+  }, []);
+
+  // Update active scene from currentTime
+  React.useEffect(() => {
+    const scenes = (window as any).__scenes;
+    if (!scenes) return;
+    let found = "";
+    for (const s of scenes) {
+      if (currentTime >= s.start && currentTime < s.end) {
+        found = s.name || "";
+        break;
+      }
+    }
+    setActiveScene(found);
+  }, [currentTime]);
+
+  // ── Save/restore position across reloads ────────────────────────────
+  const pendingSeekRef = React.useRef<number | null>(null);
+
   // Parse URL params
   const urlParams = new URLSearchParams(typeof window !== "undefined" ? window.location.search : "");
   const autoPlay = urlParams.get("autoplay") === "true";
   const startAt = parseFloat(urlParams.get("start") || urlParams.get("t") || "0") || 0;
 
   // Derive player config from data early so effects can reference them safely.
-  // These are computed on every render but only meaningful when data is set.
   const fps = data?.fps ?? 30;
   const durationInSeconds = data ? (getDurationInSeconds(data, true) || 5) : 5;
   const durationInFrames = Math.max(1, Math.ceil(durationInSeconds * fps));
 
-  const loadData = React.useCallback(() => {
-    setReady(false);
+  // ── Load data (does NOT set ready=false to avoid player unmount) ─────
+  const loadData = React.useCallback((initial = false) => {
+    if (initial) setReady(false);
     const variant = (window as any).VARIANT || "default";
     const url = variant !== "default" ? `/api/video-data?variant=${variant}` : "/api/video-data";
     fetch(url)
@@ -132,38 +171,55 @@ function PlayerApp() {
       .then((json) => {
         const root = json.root || json;
         setData(root);
-        setReady(true);
+        if (initial) setReady(true);
       })
       .catch((e) => setError(e.message));
   }, []);
 
+  // ── Initial load ────────────────────────────────────────────────────
   React.useEffect(() => {
-    loadData();
-    const handler = () => { setRefreshKey(k => k + 1); };
-    window.addEventListener("refresh-player", handler);
-    return () => { mountedRef.current = false; window.removeEventListener("refresh-player", handler); };
+    loadData(true);
+    return () => { mountedRef.current = false; };
   }, [loadData]);
 
+  // ── SSE reload: save position, load new data, restore position ──────
   React.useEffect(() => {
-    if (refreshKey > 0) loadData();
-  }, [refreshKey, loadData]);
+    const handler = () => {
+      // Save current position before reload
+      if (playerRef.current) {
+        pendingSeekRef.current = playerRef.current.getCurrentFrame();
+      }
+      loadData(false);
+    };
+    window.addEventListener("refresh-player", handler);
+    return () => window.removeEventListener("refresh-player", handler);
+  }, [loadData]);
 
-  // Seek to startAt once player is mounted
+  // ── After data loads, seek to saved position or startAt ─────────────
   React.useEffect(() => {
-    if (!ready || !data || seekAttemptedRef.current) return;
-    if (startAt > 0 && playerRef.current) {
-      // Small delay to let the player fully initialize
+    if (!ready || !data || !playerRef.current) return;
+    const targetFrame = pendingSeekRef.current ?? Math.round(startAt * fps);
+    if (targetFrame > 0) {
       const timer = setTimeout(() => {
         if (!mountedRef.current || !playerRef.current) return;
-        const frame = Math.round(startAt * fps);
-        playerRef.current.seekTo(frame);
-        seekAttemptedRef.current = true;
+        playerRef.current.seekTo(targetFrame);
+        pendingSeekRef.current = null;
       }, 100);
       return () => clearTimeout(timer);
     }
-    seekAttemptedRef.current = true;
+    pendingSeekRef.current = null;
   }, [ready, data, startAt, fps]);
 
+  // ── onFrameUpdate: track current time ───────────────────────────────
+  const handleFrameUpdate = React.useCallback((frame: number) => {
+    currentFrameRef.current = frame;
+    // Throttle state updates to ~2fps for scene tracking
+    setCurrentTime(prev => {
+      const newTime = frame / (data?.fps ?? 30);
+      return Math.abs(newTime - prev) > 0.5 ? newTime : prev;
+    });
+  }, [data?.fps]);
+
   // Keyboard shortcuts
   React.useEffect(() => {
     if (!ready || !playerRef.current) return;
@@ -300,52 +356,120 @@ function PlayerApp() {
     return () => window.removeEventListener("keydown", onKey);
   }, [ready, data, fps, durationInFrames, volume]);
 
-  // Expose seek API for external scripts
+  // Determine mode from global (set via HTML by the server)
+  const mode: string =
+    (typeof window !== "undefined" ? (window as any).MODE : null) || "preview";
+
+  // Shared state for header info
+  const [editStatus, setEditStatus] = React.useState("");
+  const [sseConnected, setSseConnected] = React.useState(false);
+  const [labelSceneInfo, setLabelSceneInfo] = React.useState("");
+
+  // SSE connection — shared across all modes as a server-liveness monitor
+  // Edit mode also listens for "reload" messages (auto-refresh on file change)
+  const suppressReloadRef = React.useRef(false);
   React.useEffect(() => {
-    if (!data) return;
-    (window as any).__remotionSeekTo = (timeInSeconds: number) => {
-      const frame = Math.round(timeInSeconds * fps);
-      playerRef.current?.seekTo(frame);
+    let evtSource: EventSource | null = null;
+    try {
+      evtSource = new EventSource("/api/events");
+      evtSource.onopen = () => setSseConnected(true);
+      evtSource.onmessage = (e: MessageEvent) => {
+        try {
+          const msg = JSON.parse(e.data);
+          if (msg.type === "reload" && !suppressReloadRef.current) {
+            window.dispatchEvent(new Event("refresh-player"));
+          }
+        } catch {}
+      };
+      evtSource.onerror = () => setSseConnected(false);
+    } catch {
+      setSseConnected(false);
+    }
+    return () => {
+      evtSource?.close();
+      setSseConnected(false);
     };
-  });
+  }, []);
 
   if (error) {
-    return React.createElement("div", {
-      style: { color: "red", padding: 40, fontFamily: "sans-serif" },
-    }, "Error: " + error);
+    return <div style={{ color: "red", padding: 40, fontFamily: "sans-serif" }}>Error: {error}</div>;
   }
 
   if (!ready) {
-    return React.createElement("div", {
-      style: { color: "#888", padding: 40, fontFamily: "sans-serif" },
-    }, "Loading...");
+    return <div style={{ color: "#888", padding: 40, fontFamily: "sans-serif" }}>Loading...</div>;
   }
 
   const width = data.width || 1080;
   const height = data.height || 1920;
 
-  return React.createElement("div", {
-    style: { width: "100%", height: "100%", background: "#000" },
-  },
-    React.createElement(Player, {
-      ref: playerRef,
-      component: MarkCut,
-      inputProps: {
-        root: data,
-        compose: {},
-      },
-      durationInFrames,
-      fps,
-      compositionWidth: width,
-      compositionHeight: height,
-      style: { width: "100%", height: "100%" },
-      controls: true,
-      showPlaybackRateControl: true,
-      allowFullscreen: true,
-      clickToPlay: false,
-      doubleClickToFullscreen: true,
-      autoPlay: autoPlay,
-    })
+  return (
+    <div
+      style={{
+        width: "100%", height: "100%", background: "#0a0a0a",
+        display: "flex", flexDirection: "column", alignItems: "center",
+      }}
+    >
+      {/* ── Header (close button + mode info) ── */}
+      <HeaderBar
+        mode={mode}
+        editStatus={editStatus}
+        sseConnected={sseConnected}
+        sceneInfo={mode === "label" ? labelSceneInfo : undefined}
+      />
+
+      {/* ── Variant switcher ── */}
+      <VariantBar />
+
+      {/* ── Player frame ── */}
+      <div id="player-frame" style={{ flex: 1, width: "100%", maxWidth: 480, minHeight: 0 }}>
+        <Player
+          ref={playerRef}
+          component={MarkCut}
+          inputProps={{ root: data, compose: {} }}
+          durationInFrames={durationInFrames}
+          fps={fps}
+          compositionWidth={width}
+          compositionHeight={height}
+          style={{ width: "100%", height: "100%" }}
+          controls={true}
+          showPlaybackRateControl={true}
+          allowFullscreen={true}
+          clickToPlay={false}
+          doubleClickToFullscreen={true}
+          autoPlay={autoPlay}
+          onFrameUpdate={handleFrameUpdate}
+        />
+      </div>
+
+      {/* ── Scene thumbnails (shared across all modes) ── */}
+      <SceneThumbnails
+        currentTime={currentTime}
+        onSeek={(t) => {
+          if (playerRef.current) {
+            const frame = Math.round(t * (data?.fps ?? 30));
+            playerRef.current.seekTo(frame);
+          }
+        }}
+      />
+
+      {/* ── Mode-specific controls ── */}
+      {mode === "edit" && (
+        <EditControls
+          onStatusChange={setEditStatus}
+          suppressReloadRef={suppressReloadRef}
+          currentTime={currentTime}
+          activeScene={activeScene}
+        />
+      )}
+      {mode === "label" && (
+        <LabelControls
+          playerRef={playerRef}
+          currentTime={currentTime}
+          onSceneChange={setLabelSceneInfo}
+        />
+      )}
+      {/* Preview mode: no extra controls */}
+    </div>
   );
 }