@bycrux/editor 0.5.1 → 0.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bycrux/editor",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "private": false,
   "type": "module",
   "exports": {

package/src/index.ts

@@ -29,6 +29,8 @@ export type {
   OverlayFactory,
   RenderEvent,
   RenderOptions,
+  CaptionEvent,
+  GenerateCaptionsOptions,
   MediaScope,
   MediaItem,
   GlobalOverlay,

package/src/types.ts

@@ -21,6 +21,7 @@ import type {
   CarouselElement,
   ImageElement,
   OverlayElement,
+  Captions,
 } from './schema'
 
 // ── Overlay compiler ─────────────────────────────────────────────────────────
@@ -68,6 +69,35 @@ export interface RenderOptions {
   scale?: number
 }
 
+// ── Caption regeneration ─────────────────────────────────────────────────────
+
+/**
+ * A single frame of caption-regeneration progress. Discriminated on `type`:
+ *  - 'log'   — a human-readable progress line.
+ *  - 'done'  — terminal success; `captions` is the freshly transcribed caption
+ *              track to patch onto the project.
+ *  - 'error' — terminal failure; `message` describes what went wrong (the host
+ *              route may emit `multi_source`/`no_clips`/`empty_keeps` — shown
+ *              verbatim).
+ */
+export type CaptionEvent =
+  | { type: 'log'; message: string }
+  | { type: 'done'; captions: Captions }
+  | { type: 'error'; message: string }
+
+/**
+ * Options for a caption-regeneration request. All optional — the host fills in
+ * sensible defaults (multilingual whisper model + auto-detected language).
+ */
+export interface GenerateCaptionsOptions {
+  /** Whisper model to run (e.g. 'large'). */
+  model?: string
+  /** Source-language hint (e.g. 'es'); omit to auto-detect. */
+  language?: string
+  /** Caption style to seed the regenerated track with. */
+  style?: string
+}
+
 // ── Overlay library types ─────────────────────────────────────────────────────
 // Copied verbatim from Montaj's `ui/src/lib/api.ts` so the package owns the
 // shape the editor consumes. A host's overlay-listing endpoints return these;
@@ -319,6 +349,17 @@ export interface EditorAdapter<P extends Project = Project> {
    * caption support omit this entirely.
    */
   resolveCaptionTemplate?(style: string): string
+
+  /**
+   * Optional: regenerate the project's caption track by re-running multilingual
+   * transcription on the host's sidecar, streaming progress as an async iterable
+   * of `CaptionEvent`s. The iterable completes after a terminal 'done' (carrying
+   * the fresh `Captions`) or 'error'. The host persists the captions server-side;
+   * the editor patches `project.captions` from the 'done' event. Hosts without a
+   * transcription pipeline omit this; the editor feature-detects its absence and
+   * hides the "Regenerate captions" control.
+   */
+  generateCaptions?(id: string, opts?: GenerateCaptionsOptions): AsyncIterable<CaptionEvent>
 }
 
 // ── Theme ────────────────────────────────────────────────────────────────────

package/src/video/CaptionRegenModal.tsx

@@ -0,0 +1,177 @@
+import { useEffect, useRef, useState } from 'react'
+import type { EditorAdapter, Project } from '../types'
+import type { Captions } from '../schema'
+
+interface CaptionRegenModalProps<P extends Project = Project> {
+  projectId: string
+  /** Adapter driving the caption-regeneration stream. Must implement
+   *  `generateCaptions` — callers gate rendering on its presence. */
+  adapter: EditorAdapter<P>
+  /** Fired on terminal success with the freshly transcribed caption track. The
+   *  caller patches `project.captions` from this; the modal then closes. */
+  onDone: (captions: Captions) => void
+  /** Fired when the modal closes (cancel, error dismiss, or post-done). */
+  onClose: () => void
+}
+
+function LogLine({ text }: { text: string }) {
+  let color = 'text-gray-400'
+  if (/ready|complete|done|transcribed/i.test(text))      color = 'text-green-400'
+  else if (/transcrib|detecting|loading|model/i.test(text)) color = 'text-sky-400'
+  else if (/extract|building|composing/i.test(text))      color = 'text-amber-400'
+  else if (/error|fail|warn/i.test(text))                 color = 'text-red-400'
+
+  return (
+    <span className={`leading-relaxed whitespace-pre-wrap break-all ${color}`}>
+      {text}
+    </span>
+  )
+}
+
+export default function CaptionRegenModal<P extends Project = Project>({ projectId, adapter, onDone, onClose }: CaptionRegenModalProps<P>) {
+  const [logs, setLogs]     = useState<string[]>([])
+  const [status, setStatus] = useState<'running' | 'done' | 'error'>('running')
+  const [errorMsg, setError] = useState<string | null>(null)
+  const logRef              = useRef<HTMLDivElement>(null)
+  const cancelledRef        = useRef(false)
+  const unmountedRef        = useRef(false)
+  const cleanupTimerRef     = useRef<ReturnType<typeof setTimeout> | null>(null)
+
+  useEffect(() => {
+    // React StrictMode in dev fires mount → cleanup → mount synchronously to
+    // catch effects that aren't idempotent. Triggering transcription is a
+    // non-idempotent effect (spawns sidecar work), so we defer the teardown in
+    // cleanup, and if the next mount fires within the same tick we rescue the
+    // pending teardown — mirroring RenderModal's StrictMode handling.
+    if (cleanupTimerRef.current !== null) {
+      clearTimeout(cleanupTimerRef.current)
+      cleanupTimerRef.current = null
+      unmountedRef.current = false
+      cancelledRef.current = false
+      return scheduleCleanup
+    }
+
+    unmountedRef.current = false
+    cancelledRef.current = false
+
+    void (async () => {
+      try {
+        for await (const ev of adapter.generateCaptions!(projectId)) {
+          if (unmountedRef.current || cancelledRef.current) break
+          if (ev.type === 'log') {
+            setLogs(l => [...l, ev.message])
+          } else if (ev.type === 'done') {
+            setStatus('done')
+            onDone(ev.captions)
+          } else {
+            setError(ev.message)
+            setStatus('error')
+          }
+        }
+      } catch (e) {
+        if (!unmountedRef.current && !cancelledRef.current) {
+          setError(e instanceof Error ? e.message : String(e))
+          setStatus('error')
+        }
+      }
+    })()
+
+    return scheduleCleanup
+
+    function scheduleCleanup() {
+      cleanupTimerRef.current = setTimeout(() => {
+        cleanupTimerRef.current = null
+        unmountedRef.current = true
+      }, 0)
+    }
+  }, [projectId, adapter, onDone])
+
+  // Auto-scroll logs
+  useEffect(() => {
+    if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight
+  }, [logs])
+
+  // Escape to close only when not running
+  useEffect(() => {
+    const onKey = (e: KeyboardEvent) => {
+      if (e.key === 'Escape' && status !== 'running') onClose()
+    }
+    document.addEventListener('keydown', onKey)
+    return () => document.removeEventListener('keydown', onKey)
+  }, [status, onClose])
+
+  function handleCancel() {
+    cancelledRef.current = true
+    onClose()
+  }
+
+  return (
+    <div className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm">
+      <div className="w-full max-w-3xl bg-gray-900 border border-gray-700 rounded-xl shadow-2xl flex flex-col overflow-hidden">
+
+        {/* Header */}
+        <div className="flex items-center justify-between px-5 py-4 border-b border-gray-800">
+          <div className="flex items-center gap-2.5">
+            {status === 'running' && <span className="w-2 h-2 rounded-full bg-amber-400 animate-pulse" />}
+            {status === 'done'    && <span className="w-2 h-2 rounded-full bg-green-400" />}
+            {status === 'error'   && <span className="w-2 h-2 rounded-full bg-red-400" />}
+            <div className="flex flex-col gap-0.5">
+              <h2 className="text-sm font-semibold text-white">
+                {status === 'running' ? 'Regenerating captions…'
+                  : status === 'done' ? 'Captions regenerated'
+                  : 'Caption regeneration failed'}
+              </h2>
+            </div>
+          </div>
+          {status !== 'running' && (
+            <button onClick={onClose} className="text-gray-500 hover:text-white transition-colors text-lg leading-none">×</button>
+          )}
+        </div>
+
+        {/* Log output */}
+        <div className="relative">
+          <button
+            onClick={() => navigator.clipboard.writeText(logs.join('\n') + (errorMsg ? '\n' + errorMsg : ''))}
+            className="absolute top-2 right-2 z-10 text-[10px] px-2 py-0.5 rounded bg-gray-800 border border-gray-700 text-gray-400 hover:text-white hover:border-gray-500 transition-colors"
+            title="Copy logs"
+          >
+            Copy
+          </button>
+          <div
+            ref={logRef}
+            className="h-96 overflow-y-auto px-4 py-3 font-mono text-[11px] text-gray-300 bg-gray-950 flex flex-col gap-0.5"
+          >
+            {logs.length === 0 && status === 'running' && (
+              <span className="text-gray-600 italic">Starting transcription…</span>
+            )}
+            {logs.map((line, i) => (
+              <LogLine key={i} text={line} />
+            ))}
+            {status === 'error' && errorMsg && (
+              <span className="text-red-400 mt-1">{errorMsg}</span>
+            )}
+          </div>
+        </div>
+
+        {/* Footer */}
+        <div className="flex items-center justify-end gap-2 px-5 py-3 border-t border-gray-800">
+          {status === 'running' ? (
+            <button
+              onClick={handleCancel}
+              className="text-sm px-4 py-1.5 rounded-md bg-gray-800 border border-gray-700 text-gray-300 hover:bg-red-900/40 hover:border-red-700 hover:text-red-300 transition-colors"
+            >
+              Cancel
+            </button>
+          ) : (
+            <button
+              onClick={onClose}
+              className="text-sm px-4 py-1.5 rounded-md bg-gray-800 border border-gray-700 text-white hover:bg-gray-700 transition-colors"
+            >
+              Close
+            </button>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}

package/src/video/VideoEditor.tsx

@@ -8,6 +8,7 @@ import Timeline from './timeline/Timeline'
 import PreviewPlayer from './preview/PreviewPlayer'
 import VersionPanel from './VersionPanel'
 import RenderModal from './RenderModal'
+import CaptionRegenModal from './CaptionRegenModal'
 
 // Generic over the host's concrete project type `P` (default = the package's
 // own `Project`). Montaj passes its richer Project; the index signature on
@@ -273,6 +274,7 @@ function ReviewSurface<P extends Project>({
   const primarySelectedId = selectedIds[0] ?? null
   const [rippleMode, setRippleMode]   = useState(false)
   const [renderOpen, setRenderOpen]   = useState(false)
+  const [regenCaptionsOpen, setRegenCaptionsOpen] = useState(false)
   // The clip/audio inspector target — derived from the timeline's inspect
   // callbacks. A Montaj-agnostic { kind, id } selector, not a project entity.
   const [inspecting, setInspecting]   = useState<{ kind: 'clip' | 'audio'; id: string } | null>(null)
@@ -471,6 +473,7 @@ function ReviewSurface<P extends Project>({
             regenEnabled={regenEnabled}
             isClipQueued={isClipQueued}
             renderSubcutRegen={renderSubcutRegen}
+            onRegenerateCaptions={adapter.generateCaptions ? () => setRegenCaptionsOpen(true) : undefined}
           />
         </div>
       </div>
@@ -509,6 +512,23 @@ function ReviewSurface<P extends Project>({
         />
       )}
 
+      {/* Caption regen modal — adapter.generateCaptions stream. On done we patch
+          project.captions via onProjectChange only. We deliberately do NOT call
+          save(): montaj persists the regenerated captions server-side and the
+          SSE subscribe frame reconciles, so a saveProject here would double-write. */}
+      {regenCaptionsOpen && adapter.generateCaptions && (
+        <CaptionRegenModal
+          adapter={adapter}
+          projectId={project.id}
+          onClose={() => setRegenCaptionsOpen(false)}
+          onDone={(captions) => {
+            const next = { ...project, captions } as P
+            onProjectChange(next)
+            setRegenCaptionsOpen(false)
+          }}
+        />
+      )}
+
       {/* Clip / audio inspector — host-rendered via render-prop seam. */}
       {inspecting && renderClipInspector?.({
         item: inspecting,

package/src/video/__tests__/CaptionRegenModal.test.tsx

@@ -0,0 +1,65 @@
+import { describe, it, expect, vi, afterEach } from 'vitest'
+import { render, screen, waitFor, cleanup } from '@testing-library/react'
+import type { CaptionEvent, EditorAdapter, ImageElement, Project } from '../../types'
+import type { Captions } from '../../schema'
+import CaptionRegenModal from '../CaptionRegenModal'
+
+afterEach(() => cleanup())
+
+const doneCaptions: Captions = {
+  style: 'pop',
+  segments: [{ text: 'hola', start: 0, end: 1, words: [] }],
+}
+
+function makeAdapter(): EditorAdapter<Project> {
+  return {
+    loadProject: vi.fn(),
+    saveProject: vi.fn(),
+    subscribe: () => () => {},
+    render: async function* () {},
+    resolveImageSrc: (el: ImageElement) => el.src,
+    compileOverlay: vi.fn(async () => () => null),
+    listGlobalOverlays: vi.fn(async () => []),
+    listSystemOverlays: vi.fn(async () => []),
+    uploadFile: vi.fn(async () => ''),
+    fileUrl: (p: string) => p,
+    generateCaptions: async function* (): AsyncIterable<CaptionEvent> {
+      yield { type: 'log', message: 'transcribing audio…' }
+      yield { type: 'done', captions: doneCaptions }
+    },
+  } as unknown as EditorAdapter<Project>
+}
+
+describe('CaptionRegenModal', () => {
+  it('streams a log line and calls onDone with the final captions', async () => {
+    const onDone = vi.fn()
+    const onClose = vi.fn()
+    render(
+      <CaptionRegenModal
+        adapter={makeAdapter()}
+        projectId="vid-1"
+        onDone={onDone}
+        onClose={onClose}
+      />,
+    )
+
+    await waitFor(() => expect(screen.getByText(/transcribing audio/i)).toBeTruthy())
+    await waitFor(() => expect(onDone).toHaveBeenCalledWith(doneCaptions))
+  })
+
+  it('shows an error message verbatim on error', async () => {
+    const errorAdapter = makeAdapter()
+    errorAdapter.generateCaptions = async function* (): AsyncIterable<CaptionEvent> {
+      yield { type: 'error', message: 'multi_source' }
+    }
+    render(
+      <CaptionRegenModal
+        adapter={errorAdapter}
+        projectId="vid-1"
+        onDone={vi.fn()}
+        onClose={vi.fn()}
+      />,
+    )
+    await waitFor(() => expect(screen.getByText('multi_source')).toBeTruthy())
+  })
+})

package/src/video/preview/useVideoPlayback.ts

@@ -50,6 +50,10 @@ export function useVideoPlayback(
   const loopOffsetRef = useRef(0)
   const rafRef        = useRef<number | null>(null)
   const rafLastMs     = useRef<number | null>(null)
+  // rAF clock for VIDEO projects — drives clip-boundary detection at ~60Hz
+  // instead of the <video> element's coarse `timeupdate` event (~4Hz). See the
+  // effect below for why.
+  const videoRafRef   = useRef<number | null>(null)
   const audioRefsMap  = useRef<Map<string, HTMLAudioElement>>(new Map())
   const audioSrcMap   = useRef<Map<string, string>>(new Map())
   // Web Audio API: GainNode per audio track allows volume > 1.0 (amplification).
@@ -693,6 +697,39 @@ export function useVideoPlayback(
     handleTimeUpdate()
   }, [handleTimeUpdate])
 
+  // ── Video boundary clock ─────────────────────────────────────────────────
+  // Drive clip-boundary detection from requestAnimationFrame (~60Hz) rather
+  // than relying on the <video> element's `timeupdate` event, which only fires
+  // ~every 250ms. Under timeupdate-gating the active clip plays up to a full
+  // ~250ms PAST its outPoint before the swap fires; on a silence-trimmed
+  // single-source timeline that overshoot is trimmed-out footage playing past
+  // the cut — the "underlying video keeps playing in the background" bug.
+  // Polling currentTime every frame tightens the boundary to ~16ms.
+  // handleTimeUpdate is idempotent (preload + swap are guarded), so the
+  // timeupdate event firing in addition to this is harmless. Canvas projects
+  // advance time via their own rAF above and are excluded here.
+  useEffect(() => {
+    if (isCanvasProject || !isPlaying) {
+      if (videoRafRef.current !== null) {
+        cancelAnimationFrame(videoRafRef.current)
+        videoRafRef.current = null
+      }
+      return
+    }
+    function pump() {
+      // The gap clock owns time during gaps; handleTimeUpdate no-ops then.
+      if (!inGapRef.current) handleTimeUpdate()
+      videoRafRef.current = requestAnimationFrame(pump)
+    }
+    videoRafRef.current = requestAnimationFrame(pump)
+    return () => {
+      if (videoRafRef.current !== null) {
+        cancelAnimationFrame(videoRafRef.current)
+        videoRafRef.current = null
+      }
+    }
+  }, [isPlaying, isCanvasProject, handleTimeUpdate])
+
   function togglePlay() {
     // GESTURE-ANCHORED: same rationale as the keydown handler — resume the
     // AudioContext synchronously inside this user-gesture call so a wired

package/src/video/timeline/Timeline.tsx

@@ -47,10 +47,14 @@ interface TimelineProps {
    *  regenQueue, storyboard, and onSave — none of which the package types know.
    *  Absent → the subcut tool is simply not rendered. */
   renderSubcutRegen?: (ctx: { clipId: string; onClose: () => void }) => ReactNode
+  /** Opens the caption-regeneration modal. Threaded down to TranscriptPanel.
+   *  Provided only when the host adapter supports `generateCaptions`; absent →
+   *  the "Regenerate" button is hidden. */
+  onRegenerateCaptions?: () => void
 }
 
 
-export default function Timeline({ project, currentTime, onTimeUpdate, onProjectChange, onCaptionEdit, onOverlayEdit, selectedIds = [], onSelectIds, onSplit, onCut, onInspectClip, onInspectAudio, rippleMode = false, getWaveformChunks, resolveFilePath, regenEnabled, isClipQueued, renderSubcutRegen }: TimelineProps) {
+export default function Timeline({ project, currentTime, onTimeUpdate, onProjectChange, onCaptionEdit, onOverlayEdit, selectedIds = [], onSelectIds, onSplit, onCut, onInspectClip, onInspectAudio, rippleMode = false, getWaveformChunks, resolveFilePath, regenEnabled, isClipQueued, renderSubcutRegen, onRegenerateCaptions }: TimelineProps) {
   const primarySelectedId = selectedIds[0] ?? null
 
   // Click/shift-click handler — additive selection on shift or meta (cmd/ctrl).
@@ -355,6 +359,7 @@ export default function Timeline({ project, currentTime, onTimeUpdate, onProject
         onCaptionEdit={onCaptionEdit}
         onProjectChange={onProjectChange}
         onExpand={() => setTranscriptModalOpen(true)}
+        onRegenerateCaptions={onRegenerateCaptions}
       />
 
       {/* ── Transcript modal ── */}

package/src/video/timeline/TranscriptPanel.tsx

@@ -10,9 +10,12 @@ interface TranscriptPanelProps {
   onCaptionEdit?: (project: Project) => void
   onProjectChange?: (project: Project) => void
   onExpand: () => void
+  /** Opens the caption-regeneration modal. Provided only when the host adapter
+   *  supports `generateCaptions`; absent → the "Regenerate" button is hidden. */
+  onRegenerateCaptions?: () => void
 }
 
-export default function TranscriptPanel({ project, captionTrack, currentTime, onCaptionEdit, onProjectChange, onExpand }: TranscriptPanelProps) {
+export default function TranscriptPanel({ project, captionTrack, currentTime, onCaptionEdit, onProjectChange, onExpand, onRegenerateCaptions }: TranscriptPanelProps) {
   const segs = captionTrack?.segments ?? []
   // Find active segment index
   const activeIdx = segs.findIndex(s => currentTime >= s.start && currentTime < s.end)
@@ -49,6 +52,14 @@ export default function TranscriptPanel({ project, captionTrack, currentTime, on
               </button>
             )
           })}
+          {onRegenerateCaptions && (
+            <button
+              className="text-[10px] text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-white border border-gray-300 dark:border-gray-700 hover:border-gray-400 dark:hover:border-gray-500 bg-gray-100 dark:bg-gray-800 hover:bg-gray-200 dark:hover:bg-gray-700 rounded px-2 py-0.5 transition-all"
+              onClick={() => onRegenerateCaptions?.()}
+            >
+              Regenerate
+            </button>
+          )}
           {segs.length > 0 && (
             <button
               className="text-[10px] text-gray-500 dark:text-gray-400 hover:text-gray-900 dark:hover:text-white border border-gray-300 dark:border-gray-700 hover:border-gray-400 dark:hover:border-gray-500 bg-gray-100 dark:bg-gray-800 hover:bg-gray-200 dark:hover:bg-gray-700 rounded px-2 py-0.5 transition-all"