@bycrux/editor 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/crop/VideoSourceCropOverlay.tsx +249 -0
- package/src/crop/__tests__/VideoSourceCropOverlay.test.tsx +105 -0
- package/src/crop/__tests__/crop-math.test.ts +15 -0
- package/src/index.ts +2 -0
- package/src/schema.ts +6 -1
- package/src/types.ts +50 -0
- package/src/video/CaptionRegenModal.tsx +177 -0
- package/src/video/RenderModal.tsx +21 -21
- package/src/video/VersionPanel.tsx +11 -11
- package/src/video/VideoEditor.tsx +148 -37
- package/src/video/__tests__/CaptionRegenModal.test.tsx +65 -0
- package/src/video/preview/PreviewPlayer.tsx +42 -4
- package/src/video/preview/__tests__/sourceCropStyle.test.ts +61 -0
- package/src/video/preview/__tests__/useVideoPlayback.test.ts +52 -0
- package/src/video/preview/sourceCropStyle.ts +70 -0
- package/src/video/preview/useVideoPlayback.ts +55 -14
- package/src/video/timeline/Timeline.tsx +6 -1
- package/src/video/timeline/TranscriptPanel.tsx +12 -1
package/package.json
CHANGED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import * as React from 'react'
|
|
2
|
+
import type { VisualItem } from '../schema'
|
|
3
|
+
import {
|
|
4
|
+
renderedSourceRect,
|
|
5
|
+
fractionToWrapperPx,
|
|
6
|
+
applyCropHandleDrag,
|
|
7
|
+
type CropFraction,
|
|
8
|
+
type CropHandle,
|
|
9
|
+
} from './crop-math'
|
|
10
|
+
|
|
11
|
+
// Sibling of CanvasCropOverlay, typed to a tracks[0] VisualItem (video) instead
|
|
12
|
+
// of an ImageElement. The full source clip is letterboxed into the preview via a
|
|
13
|
+
// muted <video>, an 8-handle free-form crop window is drawn on top, and handle
|
|
14
|
+
// drags emit the updated `sourceCrop` as source fractions clamped to [0, 1].
|
|
15
|
+
//
|
|
16
|
+
// Source dims come either from the item (sourceWidth/sourceHeight, set on a prior
|
|
17
|
+
// crop) or from the <video>'s videoWidth/videoHeight on loadedmetadata — exactly
|
|
18
|
+
// analogous to CanvasCropOverlay reading naturalWidth/Height from <img> onLoad.
|
|
19
|
+
|
|
20
|
+
const CROP_HANDLES: CropHandle[] = ['nw', 'n', 'ne', 'w', 'e', 'sw', 's', 'se']
|
|
21
|
+
|
|
22
|
+
const HANDLE_OFFSET: Record<CropHandle, { dx: 0 | 0.5 | 1; dy: 0 | 0.5 | 1; cursor: string }> = {
|
|
23
|
+
nw: { dx: 0, dy: 0, cursor: 'nw-resize' },
|
|
24
|
+
n: { dx: 0.5, dy: 0, cursor: 'n-resize' },
|
|
25
|
+
ne: { dx: 1, dy: 0, cursor: 'ne-resize' },
|
|
26
|
+
w: { dx: 0, dy: 0.5, cursor: 'w-resize' },
|
|
27
|
+
e: { dx: 1, dy: 0.5, cursor: 'e-resize' },
|
|
28
|
+
sw: { dx: 0, dy: 1, cursor: 'sw-resize' },
|
|
29
|
+
s: { dx: 0.5, dy: 1, cursor: 's-resize' },
|
|
30
|
+
se: { dx: 1, dy: 1, cursor: 'se-resize' },
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const DEFAULT_CROP: CropFraction = { x: 0, y: 0, w: 1, h: 1 }
|
|
34
|
+
|
|
35
|
+
function clampFraction(c: CropFraction): CropFraction {
|
|
36
|
+
const x = Math.min(1, Math.max(0, c.x))
|
|
37
|
+
const y = Math.min(1, Math.max(0, c.y))
|
|
38
|
+
return {
|
|
39
|
+
x,
|
|
40
|
+
y,
|
|
41
|
+
w: Math.min(1 - x, Math.max(0, c.w)),
|
|
42
|
+
h: Math.min(1 - y, Math.max(0, c.h)),
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export type VideoSourceCropOverlayProps = {
|
|
47
|
+
/** The tracks[0] video item being cropped. */
|
|
48
|
+
item: VisualItem
|
|
49
|
+
/**
|
|
50
|
+
* Host-supplied resolver for the clip's preview URL. Implement on the caller
|
|
51
|
+
* side as `(item) => adapter.fileUrl(item.nobg_preview_src ?? item.src)`.
|
|
52
|
+
*/
|
|
53
|
+
resolveSrc: (item: VisualItem) => string
|
|
54
|
+
/** Wrapper width in CSS px (the rect the overlay fills — the preview area). */
|
|
55
|
+
wrapperWidth: number
|
|
56
|
+
/** Wrapper height in CSS px. */
|
|
57
|
+
wrapperHeight: number
|
|
58
|
+
/** Called once on pointer-up (drag end) with the final clamped [0,1] fractions. */
|
|
59
|
+
onChange: (next: CropFraction) => void
|
|
60
|
+
/** Called once when the <video> fires loadedmetadata with the source's intrinsic dims. */
|
|
61
|
+
onSrcDimsLoaded: (dims: { width: number; height: number }) => void
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function VideoSourceCropOverlay({
|
|
65
|
+
item,
|
|
66
|
+
resolveSrc,
|
|
67
|
+
wrapperWidth,
|
|
68
|
+
wrapperHeight,
|
|
69
|
+
onChange,
|
|
70
|
+
onSrcDimsLoaded,
|
|
71
|
+
}: VideoSourceCropOverlayProps) {
|
|
72
|
+
const srcUrl = resolveSrc(item)
|
|
73
|
+
|
|
74
|
+
// Local mirror of dims loaded from the <video> when the item doesn't already
|
|
75
|
+
// carry them, so the crop window renders as soon as metadata arrives.
|
|
76
|
+
const [loadedDims, setLoadedDims] = React.useState<{ width: number; height: number } | null>(null)
|
|
77
|
+
const srcDims =
|
|
78
|
+
item.sourceWidth && item.sourceHeight
|
|
79
|
+
? { width: item.sourceWidth, height: item.sourceHeight }
|
|
80
|
+
: loadedDims
|
|
81
|
+
|
|
82
|
+
// Prop-driven crop (source of truth when not dragging)
|
|
83
|
+
const propCrop = item.sourceCrop ?? DEFAULT_CROP
|
|
84
|
+
|
|
85
|
+
// In-flight drag crop — non-null only during an active handle drag.
|
|
86
|
+
// While dragging, the rendered window follows this local state for smoothness.
|
|
87
|
+
// On pointer-up it's cleared after calling onChange once with the final value.
|
|
88
|
+
const [liveCrop, setLiveCrop] = React.useState<CropFraction | null>(null)
|
|
89
|
+
// Ref mirror so onHandlePointerUp always reads the latest value regardless of closure age.
|
|
90
|
+
const liveCropRef = React.useRef<CropFraction | null>(null)
|
|
91
|
+
|
|
92
|
+
// The crop we actually render: live state during drag, props otherwise.
|
|
93
|
+
const localCrop = liveCrop ?? propCrop
|
|
94
|
+
|
|
95
|
+
const rendered =
|
|
96
|
+
srcDims && wrapperWidth > 0 && wrapperHeight > 0
|
|
97
|
+
? renderedSourceRect({
|
|
98
|
+
wrapperW: wrapperWidth,
|
|
99
|
+
wrapperH: wrapperHeight,
|
|
100
|
+
srcWidth: srcDims.width,
|
|
101
|
+
srcHeight: srcDims.height,
|
|
102
|
+
})
|
|
103
|
+
: null
|
|
104
|
+
const windowPx = rendered ? fractionToWrapperPx({ crop: localCrop, rendered }) : null
|
|
105
|
+
|
|
106
|
+
const dragStateRef = React.useRef<{
|
|
107
|
+
handle: CropHandle
|
|
108
|
+
startClient: { x: number; y: number }
|
|
109
|
+
startCrop: CropFraction
|
|
110
|
+
} | null>(null)
|
|
111
|
+
|
|
112
|
+
const onHandlePointerDown = (handle: CropHandle, e: React.PointerEvent<HTMLDivElement>) => {
|
|
113
|
+
e.stopPropagation()
|
|
114
|
+
e.currentTarget.setPointerCapture?.(e.pointerId)
|
|
115
|
+
dragStateRef.current = {
|
|
116
|
+
handle,
|
|
117
|
+
startClient: { x: e.clientX, y: e.clientY },
|
|
118
|
+
startCrop: propCrop,
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const onHandlePointerMove = (e: React.PointerEvent<HTMLDivElement>) => {
|
|
123
|
+
const drag = dragStateRef.current
|
|
124
|
+
if (!drag || !srcDims) return
|
|
125
|
+
const dx = e.clientX - drag.startClient.x
|
|
126
|
+
const dy = e.clientY - drag.startClient.y
|
|
127
|
+
const next = applyCropHandleDrag({
|
|
128
|
+
handle: drag.handle,
|
|
129
|
+
initialCrop: drag.startCrop,
|
|
130
|
+
deltaPx: { x: dx, y: dy },
|
|
131
|
+
wrapperW: wrapperWidth,
|
|
132
|
+
wrapperH: wrapperHeight,
|
|
133
|
+
srcWidth: srcDims.width,
|
|
134
|
+
srcHeight: srcDims.height,
|
|
135
|
+
})
|
|
136
|
+
// Accumulate in local state for smooth visual feedback — do NOT call onChange yet.
|
|
137
|
+
const clamped = clampFraction(next)
|
|
138
|
+
liveCropRef.current = clamped
|
|
139
|
+
setLiveCrop(clamped)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const onHandlePointerUp = () => {
|
|
143
|
+
const drag = dragStateRef.current
|
|
144
|
+
dragStateRef.current = null
|
|
145
|
+
const final = liveCropRef.current
|
|
146
|
+
liveCropRef.current = null
|
|
147
|
+
setLiveCrop(null)
|
|
148
|
+
if (drag && final !== null) {
|
|
149
|
+
// Call onChange exactly once with the final value.
|
|
150
|
+
onChange(final)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return (
|
|
155
|
+
<div
|
|
156
|
+
style={{ position: 'absolute', inset: 0, overflow: 'hidden', pointerEvents: 'auto' }}
|
|
157
|
+
onPointerDown={(e) => {
|
|
158
|
+
e.stopPropagation()
|
|
159
|
+
}}
|
|
160
|
+
>
|
|
161
|
+
{/* Letterbox preview of the full source clip */}
|
|
162
|
+
<video
|
|
163
|
+
src={srcUrl}
|
|
164
|
+
muted
|
|
165
|
+
playsInline
|
|
166
|
+
preload="metadata"
|
|
167
|
+
onLoadedMetadata={(e) => {
|
|
168
|
+
const v = e.currentTarget
|
|
169
|
+
const dims = { width: v.videoWidth, height: v.videoHeight }
|
|
170
|
+
setLoadedDims(dims)
|
|
171
|
+
onSrcDimsLoaded(dims)
|
|
172
|
+
}}
|
|
173
|
+
style={{
|
|
174
|
+
position: 'absolute',
|
|
175
|
+
inset: 0,
|
|
176
|
+
width: '100%',
|
|
177
|
+
height: '100%',
|
|
178
|
+
objectFit: 'contain',
|
|
179
|
+
pointerEvents: 'none',
|
|
180
|
+
}}
|
|
181
|
+
/>
|
|
182
|
+
|
|
183
|
+
{windowPx && (
|
|
184
|
+
<>
|
|
185
|
+
{/* Dark dimming mask outside the crop window */}
|
|
186
|
+
<div
|
|
187
|
+
data-testid="crop-dim"
|
|
188
|
+
style={{
|
|
189
|
+
position: 'absolute',
|
|
190
|
+
inset: 0,
|
|
191
|
+
background: 'rgba(0, 0, 0, 0.55)',
|
|
192
|
+
clipPath: `polygon(
|
|
193
|
+
0 0, 100% 0, 100% 100%, 0 100%, 0 0,
|
|
194
|
+
${windowPx.x}px ${windowPx.y}px,
|
|
195
|
+
${windowPx.x}px ${windowPx.y + windowPx.h}px,
|
|
196
|
+
${windowPx.x + windowPx.w}px ${windowPx.y + windowPx.h}px,
|
|
197
|
+
${windowPx.x + windowPx.w}px ${windowPx.y}px,
|
|
198
|
+
${windowPx.x}px ${windowPx.y}px
|
|
199
|
+
)`,
|
|
200
|
+
pointerEvents: 'none',
|
|
201
|
+
}}
|
|
202
|
+
/>
|
|
203
|
+
|
|
204
|
+
{/* Crop window border */}
|
|
205
|
+
<div
|
|
206
|
+
data-testid="crop-window"
|
|
207
|
+
style={{
|
|
208
|
+
position: 'absolute',
|
|
209
|
+
left: windowPx.x,
|
|
210
|
+
top: windowPx.y,
|
|
211
|
+
width: windowPx.w,
|
|
212
|
+
height: windowPx.h,
|
|
213
|
+
outline: '2px solid var(--editor-selection)',
|
|
214
|
+
pointerEvents: 'none',
|
|
215
|
+
}}
|
|
216
|
+
/>
|
|
217
|
+
|
|
218
|
+
{/* 8 resize handles */}
|
|
219
|
+
{CROP_HANDLES.map((handle) => {
|
|
220
|
+
const o = HANDLE_OFFSET[handle]
|
|
221
|
+
return (
|
|
222
|
+
<div
|
|
223
|
+
key={handle}
|
|
224
|
+
data-testid={`crop-handle-${handle}`}
|
|
225
|
+
onPointerDown={(e) => onHandlePointerDown(handle, e)}
|
|
226
|
+
onPointerMove={onHandlePointerMove}
|
|
227
|
+
onPointerUp={onHandlePointerUp}
|
|
228
|
+
style={{
|
|
229
|
+
position: 'absolute',
|
|
230
|
+
left: windowPx.x + o.dx * windowPx.w - 6,
|
|
231
|
+
top: windowPx.y + o.dy * windowPx.h - 6,
|
|
232
|
+
width: 12,
|
|
233
|
+
height: 12,
|
|
234
|
+
backgroundColor: '#fff',
|
|
235
|
+
border: '1.5px solid var(--editor-selection)',
|
|
236
|
+
borderRadius: 2,
|
|
237
|
+
cursor: o.cursor,
|
|
238
|
+
zIndex: 10,
|
|
239
|
+
pointerEvents: 'auto',
|
|
240
|
+
touchAction: 'none',
|
|
241
|
+
}}
|
|
242
|
+
/>
|
|
243
|
+
)
|
|
244
|
+
})}
|
|
245
|
+
</>
|
|
246
|
+
)}
|
|
247
|
+
</div>
|
|
248
|
+
)
|
|
249
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { describe, it, expect, vi, afterEach } from 'vitest'
|
|
2
|
+
import { fireEvent, render, cleanup } from '@testing-library/react'
|
|
3
|
+
import type { VisualItem } from '../../schema'
|
|
4
|
+
import { VideoSourceCropOverlay } from '../VideoSourceCropOverlay'
|
|
5
|
+
|
|
6
|
+
afterEach(() => cleanup())
|
|
7
|
+
|
|
8
|
+
// Aspect-matched fixture: wrapper 400×500, source 800×1000. The rendered source
|
|
9
|
+
// fills the wrapper with no letterbox, so crop fractions map 1:1 onto wrapper px
|
|
10
|
+
// (x over 400, y over 500). Initial crop (0.2,0.2,0.6,0.6) → px rect (80,100)–(320,400).
|
|
11
|
+
function makeItem(overrides: Partial<VisualItem> = {}): VisualItem {
|
|
12
|
+
return {
|
|
13
|
+
id: 'v1',
|
|
14
|
+
type: 'video',
|
|
15
|
+
src: '/clip.mov',
|
|
16
|
+
start: 0,
|
|
17
|
+
end: 5,
|
|
18
|
+
sourceWidth: 800,
|
|
19
|
+
sourceHeight: 1000,
|
|
20
|
+
sourceCrop: { x: 0.2, y: 0.2, w: 0.6, h: 0.6 },
|
|
21
|
+
...overrides,
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe('VideoSourceCropOverlay', () => {
|
|
26
|
+
it('renders the 8 crop handles for a video item with known source dims', () => {
|
|
27
|
+
const { getByTestId } = render(
|
|
28
|
+
<VideoSourceCropOverlay
|
|
29
|
+
item={makeItem()}
|
|
30
|
+
resolveSrc={() => '/clip.mov'}
|
|
31
|
+
wrapperWidth={400}
|
|
32
|
+
wrapperHeight={500}
|
|
33
|
+
onChange={vi.fn()}
|
|
34
|
+
onSrcDimsLoaded={vi.fn()}
|
|
35
|
+
/>,
|
|
36
|
+
)
|
|
37
|
+
for (const h of ['nw', 'n', 'ne', 'w', 'e', 'sw', 's', 'se']) {
|
|
38
|
+
expect(getByTestId(`crop-handle-${h}`)).toBeTruthy()
|
|
39
|
+
}
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
it('emits onChange exactly once on pointer-up (not per move) with the final clamped [0,1] fractions', () => {
|
|
43
|
+
const onChange = vi.fn()
|
|
44
|
+
const { getByTestId } = render(
|
|
45
|
+
<VideoSourceCropOverlay
|
|
46
|
+
item={makeItem()}
|
|
47
|
+
resolveSrc={() => '/clip.mov'}
|
|
48
|
+
wrapperWidth={400}
|
|
49
|
+
wrapperHeight={500}
|
|
50
|
+
onChange={onChange}
|
|
51
|
+
onSrcDimsLoaded={vi.fn()}
|
|
52
|
+
/>,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
const se = getByTestId('crop-handle-se')
|
|
56
|
+
// Simulate a full drag sequence: pointerdown → multiple pointermoves → pointerup.
|
|
57
|
+
// jsdom has no PointerEvent constructor, so fireEvent.pointer* drops
|
|
58
|
+
// clientX/Y. Dispatch MouseEvents under the pointer* type names instead —
|
|
59
|
+
// MouseEvent carries clientX/Y and React listens on the event type string.
|
|
60
|
+
se.dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, clientX: 320, clientY: 400 }))
|
|
61
|
+
// Multiple intermediate moves — onChange must NOT be called during these.
|
|
62
|
+
se.dispatchEvent(new MouseEvent('pointermove', { bubbles: true, clientX: 290, clientY: 400 }))
|
|
63
|
+
se.dispatchEvent(new MouseEvent('pointermove', { bubbles: true, clientX: 270, clientY: 400 }))
|
|
64
|
+
|
|
65
|
+
// onChange must not fire during move — only after release.
|
|
66
|
+
expect(onChange).not.toHaveBeenCalled()
|
|
67
|
+
|
|
68
|
+
// Release — final position: SE handle moved left 50 px from start (320 → 270).
|
|
69
|
+
se.dispatchEvent(new MouseEvent('pointerup', { bubbles: true, clientX: 270, clientY: 400 }))
|
|
70
|
+
|
|
71
|
+
// onChange called exactly once on pointer-up.
|
|
72
|
+
expect(onChange).toHaveBeenCalledTimes(1)
|
|
73
|
+
|
|
74
|
+
const next = onChange.mock.calls[0][0]
|
|
75
|
+
// dx = -50 px over a 400-px-wide rendered source = -0.125 fractions.
|
|
76
|
+
expect(next.x).toBeCloseTo(0.2)
|
|
77
|
+
expect(next.y).toBeCloseTo(0.2)
|
|
78
|
+
expect(next.w).toBeCloseTo(0.6 - 50 / 400) // 0.475
|
|
79
|
+
expect(next.h).toBeCloseTo(0.6)
|
|
80
|
+
// All emitted fractions stay within [0, 1].
|
|
81
|
+
for (const v of [next.x, next.y, next.w, next.h]) {
|
|
82
|
+
expect(v).toBeGreaterThanOrEqual(0)
|
|
83
|
+
expect(v).toBeLessThanOrEqual(1)
|
|
84
|
+
}
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('reads source dims from the <video> loadedmetadata when not supplied on the item', () => {
|
|
88
|
+
const onSrcDimsLoaded = vi.fn()
|
|
89
|
+
const { container } = render(
|
|
90
|
+
<VideoSourceCropOverlay
|
|
91
|
+
item={makeItem({ sourceWidth: undefined, sourceHeight: undefined })}
|
|
92
|
+
resolveSrc={() => '/clip.mov'}
|
|
93
|
+
wrapperWidth={400}
|
|
94
|
+
wrapperHeight={500}
|
|
95
|
+
onChange={vi.fn()}
|
|
96
|
+
onSrcDimsLoaded={onSrcDimsLoaded}
|
|
97
|
+
/>,
|
|
98
|
+
)
|
|
99
|
+
const video = container.querySelector('video') as HTMLVideoElement
|
|
100
|
+
Object.defineProperty(video, 'videoWidth', { value: 800, configurable: true })
|
|
101
|
+
Object.defineProperty(video, 'videoHeight', { value: 1000, configurable: true })
|
|
102
|
+
fireEvent.loadedMetadata(video)
|
|
103
|
+
expect(onSrcDimsLoaded).toHaveBeenCalledWith({ width: 800, height: 1000 })
|
|
104
|
+
})
|
|
105
|
+
})
|
|
@@ -55,6 +55,21 @@ describe('wrapperPxToFraction', () => {
|
|
|
55
55
|
})
|
|
56
56
|
})
|
|
57
57
|
|
|
58
|
+
describe('video-source round-trip (regression lock: helpers are element-agnostic)', () => {
|
|
59
|
+
it('12. round-trips a CropFraction through fractionToWrapperPx → wrapperPxToFraction for a 1920×1080 source in a 1080×1920 wrapper', () => {
|
|
60
|
+
// 1920×1080 landscape video rendered inside a 1080×1920 portrait wrapper.
|
|
61
|
+
// Source is much wider than the wrapper, so it letterboxes left/right.
|
|
62
|
+
const rendered = renderedSourceRect({ wrapperW: 1080, wrapperH: 1920, srcWidth: 1920, srcHeight: 1080 })
|
|
63
|
+
const orig: import('../crop-math').CropFraction = { x: 0.1, y: 0.2, w: 0.6, h: 0.5 }
|
|
64
|
+
const px = fractionToWrapperPx({ crop: orig, rendered })
|
|
65
|
+
const back = wrapperPxToFraction({ px, rendered })
|
|
66
|
+
expect(back.x).toBeCloseTo(orig.x)
|
|
67
|
+
expect(back.y).toBeCloseTo(orig.y)
|
|
68
|
+
expect(back.w).toBeCloseTo(orig.w)
|
|
69
|
+
expect(back.h).toBeCloseTo(orig.h)
|
|
70
|
+
})
|
|
71
|
+
})
|
|
72
|
+
|
|
58
73
|
import { applyCropHandleDrag } from '../crop-math'
|
|
59
74
|
|
|
60
75
|
// Standard fixture: 400x500 element, 800x1000 source — aspect-matched so the
|
package/src/index.ts
CHANGED
package/src/schema.ts
CHANGED
|
@@ -76,7 +76,11 @@ export interface VisualItem {
|
|
|
76
76
|
remove_bg?: boolean // video type only
|
|
77
77
|
nobg_src?: string // video type only — ProRes 4444 .mov for final render
|
|
78
78
|
nobg_preview_src?: string // video type only — VP9 WebM with alpha for browser preview
|
|
79
|
+
normalizedSrc?: string // derived per-window normalized cache; render/preview prefer it; src stays original
|
|
79
80
|
muted?: boolean // video type only — suppress audio in preview and render
|
|
81
|
+
sourceCrop?: { x: number; y: number; w: number; h: number } // video type only — non-destructive crop of the source clip (0–1 fractions)
|
|
82
|
+
sourceWidth?: number // video type only — intrinsic width of the source clip in pixels
|
|
83
|
+
sourceHeight?: number // video type only — intrinsic height of the source clip in pixels
|
|
80
84
|
generation?: { // ai_video only — frozen provenance from Kling generation
|
|
81
85
|
// Single-shot fields (present when multiShot is falsy).
|
|
82
86
|
sceneId?: string
|
|
@@ -186,7 +190,7 @@ export interface Slide {
|
|
|
186
190
|
export interface EditorProject {
|
|
187
191
|
id: string
|
|
188
192
|
status: 'pending' | 'storyboard_ready' | 'draft' | 'final'
|
|
189
|
-
settings: { resolution: [number, number]; fps?: number; brandKit?: string }
|
|
193
|
+
settings: { resolution: [number, number]; fps?: number; brandKit?: string; normalize?: 'eager' | 'lazy' }
|
|
190
194
|
name?: string | null
|
|
191
195
|
editingPrompt?: string
|
|
192
196
|
slides?: Slide[]
|
|
@@ -196,6 +200,7 @@ export interface EditorProject {
|
|
|
196
200
|
assets?: Asset[]
|
|
197
201
|
carousel?: { aspect: string }
|
|
198
202
|
profile?: string
|
|
203
|
+
derivedFrom?: string // ID of the source project this was derived from (e.g. clips workflow)
|
|
199
204
|
// Host-only / pipeline fields pass through at the type level.
|
|
200
205
|
[key: string]: unknown
|
|
201
206
|
}
|
package/src/types.ts
CHANGED
|
@@ -21,6 +21,7 @@ import type {
|
|
|
21
21
|
CarouselElement,
|
|
22
22
|
ImageElement,
|
|
23
23
|
OverlayElement,
|
|
24
|
+
Captions,
|
|
24
25
|
} from './schema'
|
|
25
26
|
|
|
26
27
|
// ── Overlay compiler ─────────────────────────────────────────────────────────
|
|
@@ -68,6 +69,35 @@ export interface RenderOptions {
|
|
|
68
69
|
scale?: number
|
|
69
70
|
}
|
|
70
71
|
|
|
72
|
+
// ── Caption regeneration ─────────────────────────────────────────────────────
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* A single frame of caption-regeneration progress. Discriminated on `type`:
|
|
76
|
+
* - 'log' — a human-readable progress line.
|
|
77
|
+
* - 'done' — terminal success; `captions` is the freshly transcribed caption
|
|
78
|
+
* track to patch onto the project.
|
|
79
|
+
* - 'error' — terminal failure; `message` describes what went wrong (the host
|
|
80
|
+
* route may emit `multi_source`/`no_clips`/`empty_keeps` — shown
|
|
81
|
+
* verbatim).
|
|
82
|
+
*/
|
|
83
|
+
export type CaptionEvent =
|
|
84
|
+
| { type: 'log'; message: string }
|
|
85
|
+
| { type: 'done'; captions: Captions }
|
|
86
|
+
| { type: 'error'; message: string }
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Options for a caption-regeneration request. All optional — the host fills in
|
|
90
|
+
* sensible defaults (multilingual whisper model + auto-detected language).
|
|
91
|
+
*/
|
|
92
|
+
export interface GenerateCaptionsOptions {
|
|
93
|
+
/** Whisper model to run (e.g. 'large'). */
|
|
94
|
+
model?: string
|
|
95
|
+
/** Source-language hint (e.g. 'es'); omit to auto-detect. */
|
|
96
|
+
language?: string
|
|
97
|
+
/** Caption style to seed the regenerated track with. */
|
|
98
|
+
style?: string
|
|
99
|
+
}
|
|
100
|
+
|
|
71
101
|
// ── Overlay library types ─────────────────────────────────────────────────────
|
|
72
102
|
// Copied verbatim from Montaj's `ui/src/lib/api.ts` so the package owns the
|
|
73
103
|
// shape the editor consumes. A host's overlay-listing endpoints return these;
|
|
@@ -319,6 +349,17 @@ export interface EditorAdapter<P extends Project = Project> {
|
|
|
319
349
|
* caption support omit this entirely.
|
|
320
350
|
*/
|
|
321
351
|
resolveCaptionTemplate?(style: string): string
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Optional: regenerate the project's caption track by re-running multilingual
|
|
355
|
+
* transcription on the host's sidecar, streaming progress as an async iterable
|
|
356
|
+
* of `CaptionEvent`s. The iterable completes after a terminal 'done' (carrying
|
|
357
|
+
* the fresh `Captions`) or 'error'. The host persists the captions server-side;
|
|
358
|
+
* the editor patches `project.captions` from the 'done' event. Hosts without a
|
|
359
|
+
* transcription pipeline omit this; the editor feature-detects its absence and
|
|
360
|
+
* hides the "Regenerate captions" control.
|
|
361
|
+
*/
|
|
362
|
+
generateCaptions?(id: string, opts?: GenerateCaptionsOptions): AsyncIterable<CaptionEvent>
|
|
322
363
|
}
|
|
323
364
|
|
|
324
365
|
// ── Theme ────────────────────────────────────────────────────────────────────
|
|
@@ -448,6 +489,15 @@ export interface VideoEditorProps<P extends Project = Project> {
|
|
|
448
489
|
slots?: EditorSlots
|
|
449
490
|
readOnly?: boolean
|
|
450
491
|
onBackToSetup?: () => void
|
|
492
|
+
/**
|
|
493
|
+
* Where the host's `slots.assetsPanel` is placed in the review layout:
|
|
494
|
+
* - `'right'` (default) — a sidebar column to the right of the preview/timeline.
|
|
495
|
+
* The historical Montaj-local layout; preferred when horizontal space is ample.
|
|
496
|
+
* - `'bottom'` — a full-width region stacked below the editor. Used by hosts with
|
|
497
|
+
* constrained width (e.g. the Hub editor) where vertical stacking reads better.
|
|
498
|
+
* The host chooses per deployment; the package defaults to `'right'`.
|
|
499
|
+
*/
|
|
500
|
+
assetsPlacement?: 'right' | 'bottom'
|
|
451
501
|
|
|
452
502
|
// ── Host-supplied Montaj-specific UI (render-prop seams) ──────────────────
|
|
453
503
|
// The clip/audio inspector and the subcut-regeneration tool read host-only
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { useEffect, useRef, useState } from 'react'
|
|
2
|
+
import type { EditorAdapter, Project } from '../types'
|
|
3
|
+
import type { Captions } from '../schema'
|
|
4
|
+
|
|
5
|
+
interface CaptionRegenModalProps<P extends Project = Project> {
|
|
6
|
+
projectId: string
|
|
7
|
+
/** Adapter driving the caption-regeneration stream. Must implement
|
|
8
|
+
* `generateCaptions` — callers gate rendering on its presence. */
|
|
9
|
+
adapter: EditorAdapter<P>
|
|
10
|
+
/** Fired on terminal success with the freshly transcribed caption track. The
|
|
11
|
+
* caller patches `project.captions` from this; the modal then closes. */
|
|
12
|
+
onDone: (captions: Captions) => void
|
|
13
|
+
/** Fired when the modal closes (cancel, error dismiss, or post-done). */
|
|
14
|
+
onClose: () => void
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function LogLine({ text }: { text: string }) {
|
|
18
|
+
let color = 'text-[var(--editor-text)]/60'
|
|
19
|
+
if (/ready|complete|done|transcribed/i.test(text)) color = 'text-green-400'
|
|
20
|
+
else if (/transcrib|detecting|loading|model/i.test(text)) color = 'text-sky-400'
|
|
21
|
+
else if (/extract|building|composing/i.test(text)) color = 'text-amber-400'
|
|
22
|
+
else if (/error|fail|warn/i.test(text)) color = 'text-red-400'
|
|
23
|
+
|
|
24
|
+
return (
|
|
25
|
+
<span className={`leading-relaxed whitespace-pre-wrap break-all ${color}`}>
|
|
26
|
+
{text}
|
|
27
|
+
</span>
|
|
28
|
+
)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export default function CaptionRegenModal<P extends Project = Project>({ projectId, adapter, onDone, onClose }: CaptionRegenModalProps<P>) {
|
|
32
|
+
const [logs, setLogs] = useState<string[]>([])
|
|
33
|
+
const [status, setStatus] = useState<'running' | 'done' | 'error'>('running')
|
|
34
|
+
const [errorMsg, setError] = useState<string | null>(null)
|
|
35
|
+
const logRef = useRef<HTMLDivElement>(null)
|
|
36
|
+
const cancelledRef = useRef(false)
|
|
37
|
+
const unmountedRef = useRef(false)
|
|
38
|
+
const cleanupTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
|
39
|
+
|
|
40
|
+
useEffect(() => {
|
|
41
|
+
// React StrictMode in dev fires mount → cleanup → mount synchronously to
|
|
42
|
+
// catch effects that aren't idempotent. Triggering transcription is a
|
|
43
|
+
// non-idempotent effect (spawns sidecar work), so we defer the teardown in
|
|
44
|
+
// cleanup, and if the next mount fires within the same tick we rescue the
|
|
45
|
+
// pending teardown — mirroring RenderModal's StrictMode handling.
|
|
46
|
+
if (cleanupTimerRef.current !== null) {
|
|
47
|
+
clearTimeout(cleanupTimerRef.current)
|
|
48
|
+
cleanupTimerRef.current = null
|
|
49
|
+
unmountedRef.current = false
|
|
50
|
+
cancelledRef.current = false
|
|
51
|
+
return scheduleCleanup
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
unmountedRef.current = false
|
|
55
|
+
cancelledRef.current = false
|
|
56
|
+
|
|
57
|
+
void (async () => {
|
|
58
|
+
try {
|
|
59
|
+
for await (const ev of adapter.generateCaptions!(projectId)) {
|
|
60
|
+
if (unmountedRef.current || cancelledRef.current) break
|
|
61
|
+
if (ev.type === 'log') {
|
|
62
|
+
setLogs(l => [...l, ev.message])
|
|
63
|
+
} else if (ev.type === 'done') {
|
|
64
|
+
setStatus('done')
|
|
65
|
+
onDone(ev.captions)
|
|
66
|
+
} else {
|
|
67
|
+
setError(ev.message)
|
|
68
|
+
setStatus('error')
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
} catch (e) {
|
|
72
|
+
if (!unmountedRef.current && !cancelledRef.current) {
|
|
73
|
+
setError(e instanceof Error ? e.message : String(e))
|
|
74
|
+
setStatus('error')
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
})()
|
|
78
|
+
|
|
79
|
+
return scheduleCleanup
|
|
80
|
+
|
|
81
|
+
function scheduleCleanup() {
|
|
82
|
+
cleanupTimerRef.current = setTimeout(() => {
|
|
83
|
+
cleanupTimerRef.current = null
|
|
84
|
+
unmountedRef.current = true
|
|
85
|
+
}, 0)
|
|
86
|
+
}
|
|
87
|
+
}, [projectId, adapter, onDone])
|
|
88
|
+
|
|
89
|
+
// Auto-scroll logs
|
|
90
|
+
useEffect(() => {
|
|
91
|
+
if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight
|
|
92
|
+
}, [logs])
|
|
93
|
+
|
|
94
|
+
// Escape to close only when not running
|
|
95
|
+
useEffect(() => {
|
|
96
|
+
const onKey = (e: KeyboardEvent) => {
|
|
97
|
+
if (e.key === 'Escape' && status !== 'running') onClose()
|
|
98
|
+
}
|
|
99
|
+
document.addEventListener('keydown', onKey)
|
|
100
|
+
return () => document.removeEventListener('keydown', onKey)
|
|
101
|
+
}, [status, onClose])
|
|
102
|
+
|
|
103
|
+
function handleCancel() {
|
|
104
|
+
cancelledRef.current = true
|
|
105
|
+
onClose()
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return (
|
|
109
|
+
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm">
|
|
110
|
+
<div className="w-full max-w-3xl bg-[var(--editor-surface)] border border-[var(--editor-border)] rounded-xl shadow-2xl flex flex-col overflow-hidden">
|
|
111
|
+
|
|
112
|
+
{/* Header */}
|
|
113
|
+
<div className="flex items-center justify-between px-5 py-4 border-b border-[var(--editor-border)]">
|
|
114
|
+
<div className="flex items-center gap-2.5">
|
|
115
|
+
{status === 'running' && <span className="w-2 h-2 rounded-full bg-amber-400 animate-pulse" />}
|
|
116
|
+
{status === 'done' && <span className="w-2 h-2 rounded-full bg-green-400" />}
|
|
117
|
+
{status === 'error' && <span className="w-2 h-2 rounded-full bg-red-400" />}
|
|
118
|
+
<div className="flex flex-col gap-0.5">
|
|
119
|
+
<h2 className="text-sm font-semibold text-[var(--editor-text)]">
|
|
120
|
+
{status === 'running' ? 'Regenerating captions…'
|
|
121
|
+
: status === 'done' ? 'Captions regenerated'
|
|
122
|
+
: 'Caption regeneration failed'}
|
|
123
|
+
</h2>
|
|
124
|
+
</div>
|
|
125
|
+
</div>
|
|
126
|
+
{status !== 'running' && (
|
|
127
|
+
<button onClick={onClose} className="text-[var(--editor-text)]/55 hover:text-[var(--editor-text)] transition-colors text-lg leading-none">×</button>
|
|
128
|
+
)}
|
|
129
|
+
</div>
|
|
130
|
+
|
|
131
|
+
{/* Log output */}
|
|
132
|
+
<div className="relative">
|
|
133
|
+
<button
|
|
134
|
+
onClick={() => navigator.clipboard.writeText(logs.join('\n') + (errorMsg ? '\n' + errorMsg : ''))}
|
|
135
|
+
className="absolute top-2 right-2 z-10 text-[10px] px-2 py-0.5 rounded bg-[var(--editor-surface)] border border-[var(--editor-border)] text-[var(--editor-text)]/60 hover:text-[var(--editor-text)] hover:border-[var(--editor-border)] transition-colors"
|
|
136
|
+
title="Copy logs"
|
|
137
|
+
>
|
|
138
|
+
Copy
|
|
139
|
+
</button>
|
|
140
|
+
<div
|
|
141
|
+
ref={logRef}
|
|
142
|
+
className="h-96 overflow-y-auto px-4 py-3 font-mono text-[11px] text-[var(--editor-text)]/80 bg-[var(--editor-surface)] flex flex-col gap-0.5"
|
|
143
|
+
>
|
|
144
|
+
{logs.length === 0 && status === 'running' && (
|
|
145
|
+
<span className="text-[var(--editor-text)]/40 italic">Starting transcription…</span>
|
|
146
|
+
)}
|
|
147
|
+
{logs.map((line, i) => (
|
|
148
|
+
<LogLine key={i} text={line} />
|
|
149
|
+
))}
|
|
150
|
+
{status === 'error' && errorMsg && (
|
|
151
|
+
<span className="text-red-400 mt-1">{errorMsg}</span>
|
|
152
|
+
)}
|
|
153
|
+
</div>
|
|
154
|
+
</div>
|
|
155
|
+
|
|
156
|
+
{/* Footer */}
|
|
157
|
+
<div className="flex items-center justify-end gap-2 px-5 py-3 border-t border-[var(--editor-border)]">
|
|
158
|
+
{status === 'running' ? (
|
|
159
|
+
<button
|
|
160
|
+
onClick={handleCancel}
|
|
161
|
+
className="text-sm px-4 py-1.5 rounded-md bg-[var(--editor-surface)] border border-[var(--editor-border)] text-[var(--editor-text)]/80 hover:bg-red-900/40 hover:border-red-700 hover:text-red-300 transition-colors"
|
|
162
|
+
>
|
|
163
|
+
Cancel
|
|
164
|
+
</button>
|
|
165
|
+
) : (
|
|
166
|
+
<button
|
|
167
|
+
onClick={onClose}
|
|
168
|
+
className="text-sm px-4 py-1.5 rounded-md bg-[var(--editor-surface)] border border-[var(--editor-border)] text-[var(--editor-text)] hover:opacity-90 transition-colors"
|
|
169
|
+
>
|
|
170
|
+
Close
|
|
171
|
+
</button>
|
|
172
|
+
)}
|
|
173
|
+
</div>
|
|
174
|
+
</div>
|
|
175
|
+
</div>
|
|
176
|
+
)
|
|
177
|
+
}
|