@editframe/elements 0.16.7-beta.0 → 0.17.6-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/DecoderResetFrequency.test.d.ts +1 -0
- package/dist/DecoderResetRecovery.test.d.ts +1 -0
- package/dist/DelayedLoadingState.d.ts +48 -0
- package/dist/DelayedLoadingState.integration.test.d.ts +1 -0
- package/dist/DelayedLoadingState.js +113 -0
- package/dist/DelayedLoadingState.test.d.ts +1 -0
- package/dist/EF_FRAMEGEN.d.ts +10 -1
- package/dist/EF_FRAMEGEN.js +199 -179
- package/dist/EF_INTERACTIVE.js +2 -6
- package/dist/EF_RENDERING.js +1 -3
- package/dist/JitTranscodingClient.browsertest.d.ts +1 -0
- package/dist/JitTranscodingClient.d.ts +167 -0
- package/dist/JitTranscodingClient.js +373 -0
- package/dist/JitTranscodingClient.test.d.ts +1 -0
- package/dist/LoadingDebounce.test.d.ts +1 -0
- package/dist/LoadingIndicator.browsertest.d.ts +0 -0
- package/dist/ManualScrubTest.test.d.ts +1 -0
- package/dist/ScrubResolvedFlashing.test.d.ts +1 -0
- package/dist/ScrubTrackIntegration.test.d.ts +1 -0
- package/dist/ScrubTrackManager.d.ts +96 -0
- package/dist/ScrubTrackManager.js +216 -0
- package/dist/ScrubTrackManager.test.d.ts +1 -0
- package/dist/SegmentSwitchLoading.test.d.ts +1 -0
- package/dist/VideoSeekFlashing.browsertest.d.ts +0 -0
- package/dist/VideoStuckDiagnostic.test.d.ts +1 -0
- package/dist/elements/CrossUpdateController.js +13 -15
- package/dist/elements/EFAudio.browsertest.d.ts +0 -0
- package/dist/elements/EFAudio.d.ts +1 -1
- package/dist/elements/EFAudio.js +30 -43
- package/dist/elements/EFCaptions.js +337 -373
- package/dist/elements/EFImage.js +64 -90
- package/dist/elements/EFMedia.d.ts +98 -33
- package/dist/elements/EFMedia.js +1169 -678
- package/dist/elements/EFSourceMixin.js +31 -48
- package/dist/elements/EFTemporal.d.ts +1 -0
- package/dist/elements/EFTemporal.js +266 -360
- package/dist/elements/EFTimegroup.d.ts +3 -1
- package/dist/elements/EFTimegroup.js +262 -323
- package/dist/elements/EFVideo.browsertest.d.ts +0 -0
- package/dist/elements/EFVideo.d.ts +90 -2
- package/dist/elements/EFVideo.js +408 -111
- package/dist/elements/EFWaveform.js +375 -411
- package/dist/elements/FetchMixin.js +14 -24
- package/dist/elements/MediaController.d.ts +30 -0
- package/dist/elements/TargetController.js +130 -156
- package/dist/elements/TimegroupController.js +17 -19
- package/dist/elements/durationConverter.js +15 -4
- package/dist/elements/parseTimeToMs.js +4 -10
- package/dist/elements/printTaskStatus.d.ts +2 -0
- package/dist/elements/printTaskStatus.js +11 -0
- package/dist/elements/updateAnimations.js +39 -59
- package/dist/getRenderInfo.js +58 -67
- package/dist/gui/ContextMixin.js +203 -288
- package/dist/gui/EFConfiguration.js +27 -43
- package/dist/gui/EFFilmstrip.js +440 -620
- package/dist/gui/EFFitScale.js +112 -135
- package/dist/gui/EFFocusOverlay.js +45 -61
- package/dist/gui/EFPreview.js +30 -49
- package/dist/gui/EFScrubber.js +78 -99
- package/dist/gui/EFTimeDisplay.js +49 -70
- package/dist/gui/EFToggleLoop.js +17 -34
- package/dist/gui/EFTogglePlay.js +37 -58
- package/dist/gui/EFWorkbench.js +66 -88
- package/dist/gui/TWMixin.js +2 -48
- package/dist/gui/TWMixin2.js +31 -0
- package/dist/gui/efContext.js +2 -6
- package/dist/gui/fetchContext.js +1 -3
- package/dist/gui/focusContext.js +1 -3
- package/dist/gui/focusedElementContext.js +2 -6
- package/dist/gui/playingContext.js +1 -4
- package/dist/index.js +5 -30
- package/dist/msToTimeCode.js +11 -13
- package/dist/style.css +2 -1
- package/package.json +3 -3
- package/src/elements/EFAudio.browsertest.ts +569 -0
- package/src/elements/EFAudio.ts +4 -6
- package/src/elements/EFCaptions.browsertest.ts +0 -1
- package/src/elements/EFImage.browsertest.ts +0 -1
- package/src/elements/EFMedia.browsertest.ts +147 -115
- package/src/elements/EFMedia.ts +1339 -307
- package/src/elements/EFTemporal.browsertest.ts +0 -1
- package/src/elements/EFTemporal.ts +11 -0
- package/src/elements/EFTimegroup.ts +73 -10
- package/src/elements/EFVideo.browsertest.ts +680 -0
- package/src/elements/EFVideo.ts +729 -50
- package/src/elements/EFWaveform.ts +4 -4
- package/src/elements/MediaController.ts +108 -0
- package/src/elements/__screenshots__/EFMedia.browsertest.ts/EFMedia-JIT-audio-playback-audioBufferTask-should-work-in-JIT-mode-without-URL-errors-1.png +0 -0
- package/src/elements/printTaskStatus.ts +16 -0
- package/src/elements/updateAnimations.ts +6 -0
- package/src/gui/TWMixin.ts +10 -3
- package/test/EFVideo.frame-tasks.browsertest.ts +524 -0
- package/test/EFVideo.framegen.browsertest.ts +118 -0
- package/test/createJitTestClips.ts +293 -0
- package/test/useAssetMSW.ts +49 -0
- package/test/useMSW.ts +31 -0
- package/types.json +1 -1
- package/dist/gui/TWMixin.css.js +0 -4
- /package/dist/elements/{TargetController.test.d.ts → TargetController.browsertest.d.ts} +0 -0
- /package/src/elements/{TargetController.test.ts → TargetController.browsertest.ts} +0 -0
package/src/elements/EFVideo.ts
CHANGED
|
@@ -1,10 +1,27 @@
|
|
|
1
|
+
import { VideoAsset } from "@editframe/assets/EncodedAsset.js";
|
|
1
2
|
import { Task } from "@lit/task";
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
3
|
+
import debug from "debug";
|
|
4
|
+
import { css, html, type PropertyValueMap } from "lit";
|
|
5
|
+
import { customElement, state } from "lit/decorators.js";
|
|
4
6
|
import { createRef, ref } from "lit/directives/ref.js";
|
|
5
|
-
|
|
7
|
+
import { DelayedLoadingState } from "../DelayedLoadingState.js";
|
|
6
8
|
import { TWMixin } from "../gui/TWMixin.js";
|
|
9
|
+
import { type CacheStats, ScrubTrackManager } from "../ScrubTrackManager.js";
|
|
7
10
|
import { EFMedia } from "./EFMedia.js";
|
|
11
|
+
import { printTaskStatus } from "./printTaskStatus.ts";
|
|
12
|
+
|
|
13
|
+
// EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
|
|
14
|
+
declare global {
|
|
15
|
+
var EF_FRAMEGEN: import("../EF_FRAMEGEN.js").EFFramegen;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const log = debug("ef:elements:EFVideo");
|
|
19
|
+
|
|
20
|
+
interface LoadingState {
|
|
21
|
+
isLoading: boolean;
|
|
22
|
+
operation: "scrub-segment" | "video-segment" | "seeking" | "decoding" | null;
|
|
23
|
+
message: string;
|
|
24
|
+
}
|
|
8
25
|
|
|
9
26
|
@customElement("ef-video")
|
|
10
27
|
export class EFVideo extends TWMixin(EFMedia) {
|
|
@@ -15,6 +32,7 @@ export class EFVideo extends TWMixin(EFMedia) {
|
|
|
15
32
|
css`
|
|
16
33
|
:host {
|
|
17
34
|
display: block;
|
|
35
|
+
position: relative;
|
|
18
36
|
}
|
|
19
37
|
canvas {
|
|
20
38
|
all: inherit;
|
|
@@ -29,12 +47,105 @@ export class EFVideo extends TWMixin(EFMedia) {
|
|
|
29
47
|
outline: none;
|
|
30
48
|
box-shadow: none;
|
|
31
49
|
}
|
|
50
|
+
.loading-overlay {
|
|
51
|
+
position: absolute;
|
|
52
|
+
top: 0;
|
|
53
|
+
left: 0;
|
|
54
|
+
right: 0;
|
|
55
|
+
bottom: 0;
|
|
56
|
+
background: rgba(0, 0, 0, 0.6);
|
|
57
|
+
display: flex;
|
|
58
|
+
align-items: center;
|
|
59
|
+
justify-content: center;
|
|
60
|
+
z-index: 10;
|
|
61
|
+
backdrop-filter: blur(2px);
|
|
62
|
+
}
|
|
63
|
+
.loading-content {
|
|
64
|
+
background: rgba(0, 0, 0, 0.8);
|
|
65
|
+
border-radius: 8px;
|
|
66
|
+
padding: 16px 24px;
|
|
67
|
+
display: flex;
|
|
68
|
+
align-items: center;
|
|
69
|
+
gap: 12px;
|
|
70
|
+
color: white;
|
|
71
|
+
font-size: 14px;
|
|
72
|
+
font-weight: 500;
|
|
73
|
+
}
|
|
74
|
+
.loading-spinner {
|
|
75
|
+
width: 20px;
|
|
76
|
+
height: 20px;
|
|
77
|
+
border: 2px solid rgba(255, 255, 255, 0.2);
|
|
78
|
+
border-left: 2px solid #fff;
|
|
79
|
+
border-radius: 50%;
|
|
80
|
+
animation: spin 1s linear infinite;
|
|
81
|
+
}
|
|
82
|
+
@keyframes spin {
|
|
83
|
+
0% { transform: rotate(0deg); }
|
|
84
|
+
100% { transform: rotate(360deg); }
|
|
85
|
+
}
|
|
86
|
+
.loading-message {
|
|
87
|
+
font-size: 12px;
|
|
88
|
+
opacity: 0.8;
|
|
89
|
+
}
|
|
32
90
|
`,
|
|
33
91
|
];
|
|
34
92
|
canvasRef = createRef<HTMLCanvasElement>();
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Scrub track manager for fast timeline navigation
|
|
96
|
+
*/
|
|
97
|
+
scrubTrackManager?: ScrubTrackManager;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Track last seek time for fast seeking detection
|
|
101
|
+
*/
|
|
102
|
+
private lastSeekTimeMs = 0;
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Delayed loading state manager for user feedback
|
|
106
|
+
*/
|
|
107
|
+
private delayedLoadingState: DelayedLoadingState;
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Loading state for user feedback
|
|
111
|
+
*/
|
|
112
|
+
@state()
|
|
113
|
+
loadingState = {
|
|
114
|
+
isLoading: false,
|
|
115
|
+
operation: null as LoadingState["operation"],
|
|
116
|
+
message: "",
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
constructor() {
|
|
120
|
+
super();
|
|
121
|
+
|
|
122
|
+
// Initialize delayed loading state with callback to update UI
|
|
123
|
+
this.delayedLoadingState = new DelayedLoadingState(
|
|
124
|
+
250,
|
|
125
|
+
(isLoading, message) => {
|
|
126
|
+
this.setLoadingState(isLoading, null, message);
|
|
127
|
+
},
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
35
131
|
render() {
|
|
36
132
|
return html`
|
|
37
133
|
<canvas ${ref(this.canvasRef)}></canvas>
|
|
134
|
+
${
|
|
135
|
+
this.loadingState.isLoading
|
|
136
|
+
? html`
|
|
137
|
+
<div class="loading-overlay">
|
|
138
|
+
<div class="loading-content">
|
|
139
|
+
<div class="loading-spinner"></div>
|
|
140
|
+
<div>
|
|
141
|
+
<div>Loading Video...</div>
|
|
142
|
+
<div class="loading-message">${this.loadingState.message}</div>
|
|
143
|
+
</div>
|
|
144
|
+
</div>
|
|
145
|
+
</div>
|
|
146
|
+
`
|
|
147
|
+
: ""
|
|
148
|
+
}
|
|
38
149
|
`;
|
|
39
150
|
}
|
|
40
151
|
|
|
@@ -46,85 +157,653 @@ export class EFVideo extends TWMixin(EFMedia) {
|
|
|
46
157
|
// If frames are fed in out of order, the decoder may crash.
|
|
47
158
|
#decoderLock = false;
|
|
48
159
|
|
|
160
|
+
// Track if decoder needs reset due to errors
|
|
161
|
+
#decoderNeedsReset = false;
|
|
162
|
+
|
|
49
163
|
frameTask = new Task(this, {
|
|
50
|
-
args: () =>
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
this.fetchSeekTask.status,
|
|
56
|
-
this.videoAssetTask.status,
|
|
57
|
-
this.paintTask.status,
|
|
58
|
-
] as const,
|
|
59
|
-
task: async () => {
|
|
60
|
-
await this.trackFragmentIndexLoader.taskComplete;
|
|
61
|
-
await this.initSegmentsLoader.taskComplete;
|
|
164
|
+
args: () => [this.desiredSeekTimeMs] as const,
|
|
165
|
+
onError: (error) => {
|
|
166
|
+
console.error("frameTask error", error);
|
|
167
|
+
},
|
|
168
|
+
task: async ([_desiredSeekTimeMs], { signal }) => {
|
|
62
169
|
await this.seekTask.taskComplete;
|
|
63
|
-
|
|
170
|
+
if (signal.aborted) {
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
await this.fragmentIndexTask.taskComplete;
|
|
174
|
+
if (signal.aborted) {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
await this.mediaSegmentsTask.taskComplete;
|
|
178
|
+
if (signal.aborted) {
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
64
181
|
await this.videoAssetTask.taskComplete;
|
|
182
|
+
if (signal.aborted) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
65
185
|
await this.paintTask.taskComplete;
|
|
186
|
+
if (signal.aborted) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
get frameTaskStatus() {
|
|
193
|
+
return {
|
|
194
|
+
desiredSeekTimeMs: this.desiredSeekTimeMs,
|
|
195
|
+
fragmentIndexTask: printTaskStatus(this.fragmentIndexTask.status),
|
|
196
|
+
seekTask: printTaskStatus(this.seekTask.status),
|
|
197
|
+
mediaSegmentsTask: printTaskStatus(this.mediaSegmentsTask.status),
|
|
198
|
+
assetSegmentLoader: printTaskStatus(this.assetSegmentLoader.status),
|
|
199
|
+
assetSegmentKeysTask: printTaskStatus(this.assetSegmentKeysTask.status),
|
|
200
|
+
assetInitSegmentsTask: printTaskStatus(this.assetInitSegmentsTask.status),
|
|
201
|
+
videoAssetTask: printTaskStatus(this.videoAssetTask.status),
|
|
202
|
+
paintTask: printTaskStatus(this.paintTask.status),
|
|
203
|
+
frameTask: printTaskStatus(this.frameTask.status),
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
#lastVideoAsset: any = null;
|
|
208
|
+
|
|
209
|
+
protected updated(
|
|
210
|
+
changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
|
|
211
|
+
): void {
|
|
212
|
+
super.updated(changedProperties);
|
|
213
|
+
|
|
214
|
+
const currentVideoAsset = this.videoAssetTask.value;
|
|
215
|
+
if (currentVideoAsset !== this.#lastVideoAsset) {
|
|
216
|
+
// Track video asset changes for reference, but don't reset decoder
|
|
217
|
+
// Decoder resets should only happen due to actual decoder errors, not normal asset transitions
|
|
218
|
+
this.#lastVideoAsset = currentVideoAsset;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Initialize scrub track manager for JIT transcode mode
|
|
222
|
+
this.initializeScrubTrackManager();
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Initialize scrub track manager if needed
|
|
227
|
+
*/
|
|
228
|
+
private async initializeScrubTrackManager(): Promise<void> {
|
|
229
|
+
const mode = this.effectiveMode;
|
|
230
|
+
|
|
231
|
+
// Only initialize for JIT transcode mode with valid src
|
|
232
|
+
if (mode === "jit-transcode" && this.src && !this.scrubTrackManager) {
|
|
233
|
+
const jitClient = this.jitClientTask.value;
|
|
234
|
+
if (jitClient) {
|
|
235
|
+
try {
|
|
236
|
+
this.scrubTrackManager = new ScrubTrackManager(this.src, jitClient, {
|
|
237
|
+
onLoadingStateChange: (isLoading: boolean, message?: string) => {
|
|
238
|
+
if (isLoading) {
|
|
239
|
+
// Only show loading for user-visible operations (non-background)
|
|
240
|
+
this.startDelayedLoading(
|
|
241
|
+
"scrub-segment",
|
|
242
|
+
message || "Loading scrub track...",
|
|
243
|
+
);
|
|
244
|
+
} else {
|
|
245
|
+
this.clearDelayedLoading("scrub-segment");
|
|
246
|
+
}
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
await this.scrubTrackManager.initialize();
|
|
251
|
+
} catch (error) {
|
|
252
|
+
console.warn("Failed to initialize scrub track manager:", error);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Start a delayed loading operation for testing
|
|
260
|
+
*/
|
|
261
|
+
startDelayedLoading(
|
|
262
|
+
operationId: string,
|
|
263
|
+
message: string,
|
|
264
|
+
options: { background?: boolean } = {},
|
|
265
|
+
): void {
|
|
266
|
+
this.delayedLoadingState.startLoading(operationId, message, options);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Clear a delayed loading operation for testing
|
|
271
|
+
*/
|
|
272
|
+
clearDelayedLoading(operationId: string): void {
|
|
273
|
+
this.delayedLoadingState.clearLoading(operationId);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Set loading state for user feedback
|
|
278
|
+
*/
|
|
279
|
+
private setLoadingState(
|
|
280
|
+
isLoading: boolean,
|
|
281
|
+
operation: LoadingState["operation"] = null,
|
|
282
|
+
message = "",
|
|
283
|
+
): void {
|
|
284
|
+
this.loadingState = {
|
|
285
|
+
isLoading,
|
|
286
|
+
operation,
|
|
287
|
+
message,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
videoAssetTask = new Task(this, {
|
|
292
|
+
autoRun: true,
|
|
293
|
+
args: () => [this.effectiveMode, this.mediaSegmentsTask.value] as const,
|
|
294
|
+
onError: (error) => {
|
|
295
|
+
console.error("videoAsset task error", error);
|
|
296
|
+
},
|
|
297
|
+
task: async ([mode, _files], { signal: _signal }) => {
|
|
298
|
+
await this.mediaSegmentsTask.taskComplete;
|
|
299
|
+
if (_signal.aborted) {
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
await this.fragmentIndexTask.taskComplete;
|
|
304
|
+
if (_signal.aborted) {
|
|
305
|
+
return undefined;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Get fresh values
|
|
309
|
+
const files = this.mediaSegmentsTask.value;
|
|
310
|
+
const fragmentIndex = this.fragmentIndexTask.value;
|
|
311
|
+
|
|
312
|
+
if (!files) {
|
|
313
|
+
log("trace: videoAsset task aborted - no files");
|
|
314
|
+
throw new Error(
|
|
315
|
+
`Video asset creation failed: No media segment files available. This indicates a problem with media segment loading for source: "${this.src}"`,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const computedVideoTrackId = Object.values(fragmentIndex ?? {}).find(
|
|
320
|
+
(track) => track.type === "video",
|
|
321
|
+
)?.track;
|
|
322
|
+
|
|
323
|
+
if (computedVideoTrackId === undefined) {
|
|
324
|
+
log("trace: videoAsset task aborted - no video track");
|
|
325
|
+
throw new Error(
|
|
326
|
+
`Video asset creation failed: No video track found in media segments. Source may not contain video content: "${this.src}"`,
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const videoFile = files[computedVideoTrackId];
|
|
331
|
+
if (!videoFile) {
|
|
332
|
+
log("trace: videoAsset task aborted - no video file");
|
|
333
|
+
throw new Error(
|
|
334
|
+
`Video asset creation failed: Video file not available for track ${computedVideoTrackId}. Media segment loading may have failed for source: "${this.src}"`,
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Cleanup existing asset
|
|
339
|
+
const existingAsset = this.videoAssetTask.value;
|
|
340
|
+
if (existingAsset) {
|
|
341
|
+
for (const frame of existingAsset?.decodedFrames || []) {
|
|
342
|
+
frame.close();
|
|
343
|
+
}
|
|
344
|
+
const decoder = existingAsset?.videoDecoder;
|
|
345
|
+
if (decoder && decoder.state !== "closed") {
|
|
346
|
+
decoder.close();
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
if (_signal.aborted) {
|
|
351
|
+
return undefined;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
log("trace: creating video asset", { mode });
|
|
355
|
+
|
|
356
|
+
// Get start time offset from fragment index (timing correction for FFmpeg processing)
|
|
357
|
+
const videoTrackFragmentIndex = Object.values(fragmentIndex ?? {}).find(
|
|
358
|
+
(track) => track.type === "video",
|
|
359
|
+
);
|
|
360
|
+
const startTimeOffsetMs = Number(
|
|
361
|
+
(videoTrackFragmentIndex?.startTimeOffsetMs ?? 0).toFixed(5),
|
|
362
|
+
);
|
|
363
|
+
|
|
364
|
+
// Single branching point for creation method
|
|
365
|
+
if (mode === "jit-transcode") {
|
|
366
|
+
const result = await VideoAsset.createFromCompleteMP4(
|
|
367
|
+
`jit-segment-${computedVideoTrackId}`,
|
|
368
|
+
videoFile,
|
|
369
|
+
{ startTimeOffsetMs },
|
|
370
|
+
);
|
|
371
|
+
return result;
|
|
372
|
+
}
|
|
373
|
+
const result = await VideoAsset.createFromReadableStream(
|
|
374
|
+
"video.mp4",
|
|
375
|
+
videoFile.stream(),
|
|
376
|
+
videoFile,
|
|
377
|
+
{ startTimeOffsetMs },
|
|
378
|
+
);
|
|
379
|
+
return result;
|
|
66
380
|
},
|
|
67
381
|
});
|
|
68
382
|
|
|
69
383
|
paintTask = new Task(this, {
|
|
70
|
-
args: () => [this.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
384
|
+
args: () => [this.desiredSeekTimeMs] as const,
|
|
385
|
+
onError: (error) => {
|
|
386
|
+
console.error("paintTask error", error);
|
|
387
|
+
},
|
|
388
|
+
task: async ([_seekToMs], { signal }) => {
|
|
389
|
+
// Check if we're in production rendering mode vs preview mode
|
|
390
|
+
const isProductionRendering = this.isInProductionRenderingMode();
|
|
391
|
+
|
|
392
|
+
// EF_FRAMEGEN-aware rendering mode detection
|
|
393
|
+
if (!isProductionRendering) {
|
|
394
|
+
// Preview mode: skip rendering during initialization to prevent artifacts
|
|
395
|
+
if (
|
|
396
|
+
!this.rootTimegroup ||
|
|
397
|
+
(this.rootTimegroup.currentTimeMs === 0 &&
|
|
398
|
+
this.desiredSeekTimeMs === 0)
|
|
399
|
+
) {
|
|
400
|
+
return; // Skip initialization frame in preview mode
|
|
401
|
+
}
|
|
402
|
+
// Preview mode: proceed with rendering
|
|
403
|
+
} else {
|
|
404
|
+
// Production rendering mode: only render when EF_FRAMEGEN has explicitly started frame rendering
|
|
405
|
+
// This prevents initialization frames before the actual render sequence begins
|
|
406
|
+
if (!this.rootTimegroup) {
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
if (!this.isFrameRenderingActive()) {
|
|
411
|
+
return; // Wait for EF_FRAMEGEN to start frame sequence
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Production mode: EF_FRAMEGEN has started frame sequence, proceed with rendering
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
if (signal.aborted) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// CRITICAL: For segment transitions, ensure we wait for the correct mediaSegmentsTask
|
|
422
|
+
// This prevents using stale VideoAssets from previous segments
|
|
423
|
+
|
|
424
|
+
await this.mediaSegmentsTask.taskComplete;
|
|
425
|
+
if (signal.aborted) {
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// CRITICAL: Always await fresh videoAsset just before using it
|
|
430
|
+
// This prevents race conditions where old VideoAssets with closed decoders are used
|
|
431
|
+
await this.videoAssetTask.taskComplete;
|
|
432
|
+
if (signal.aborted) {
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// Get fresh values after await - ensures we use current VideoAsset
|
|
437
|
+
const videoAsset = this.videoAssetTask.value;
|
|
438
|
+
const currentSeekToMs = this.desiredSeekTimeMs; // Use current seek time, not captured
|
|
439
|
+
|
|
78
440
|
if (!videoAsset) {
|
|
441
|
+
log("trace: paintTask aborted - no video asset");
|
|
442
|
+
throw new Error(
|
|
443
|
+
`Frame rendering failed: No video asset available. This may indicate a problem with video loading or an invalid source: "${this.src}"`,
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Check if decoder needs reset due to previous errors
|
|
448
|
+
if (this.#decoderNeedsReset) {
|
|
449
|
+
try {
|
|
450
|
+
// Reset the video decoder
|
|
451
|
+
if (videoAsset?.videoDecoder) {
|
|
452
|
+
videoAsset.configureDecoder();
|
|
453
|
+
} else {
|
|
454
|
+
console.warn("No video decoder available for reset");
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Clear the flag after successful reset
|
|
458
|
+
this.#decoderNeedsReset = false;
|
|
459
|
+
} catch (resetError) {
|
|
460
|
+
console.error("reset error", resetError);
|
|
461
|
+
// Keep the flag set if reset fails
|
|
462
|
+
throw new Error(
|
|
463
|
+
`Frame rendering failed: Unable to reset video decoder after previous error. Decoder state: ${resetError instanceof Error ? resetError.message : "Unknown error"}. Try refreshing the page or reloading the video.`,
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
if (signal.aborted) {
|
|
79
468
|
return;
|
|
80
469
|
}
|
|
470
|
+
|
|
81
471
|
if (this.#decoderLock) {
|
|
82
472
|
return;
|
|
83
473
|
}
|
|
474
|
+
|
|
84
475
|
try {
|
|
85
476
|
this.#decoderLock = true;
|
|
86
|
-
const frame = await videoAsset.seekToTime(seekToMs / 1000);
|
|
87
477
|
|
|
88
|
-
|
|
89
|
-
|
|
478
|
+
// Validate VideoAsset is still current and decoder is in valid state
|
|
479
|
+
const currentVideoAsset = this.videoAssetTask.value;
|
|
480
|
+
if (videoAsset !== currentVideoAsset) {
|
|
481
|
+
return; // Skip render with stale videoAsset
|
|
90
482
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
483
|
+
|
|
484
|
+
// Check decoder state before using it
|
|
485
|
+
const decoderState = videoAsset?.videoDecoder?.state;
|
|
486
|
+
if (decoderState === "closed") {
|
|
487
|
+
return; // Skip render with closed decoder
|
|
94
488
|
}
|
|
95
489
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
this.
|
|
100
|
-
|
|
101
|
-
this.
|
|
102
|
-
|
|
490
|
+
// Try scrub track first for JIT transcode mode
|
|
491
|
+
if (this.effectiveMode === "jit-transcode" && this.scrubTrackManager) {
|
|
492
|
+
const shouldUseScrub =
|
|
493
|
+
this.scrubTrackManager.shouldUseScrubTrack(currentSeekToMs);
|
|
494
|
+
const isFastSeeking = this.scrubTrackManager.isFastSeeking(
|
|
495
|
+
this.lastSeekTimeMs,
|
|
496
|
+
currentSeekToMs,
|
|
497
|
+
);
|
|
498
|
+
|
|
499
|
+
if (false || shouldUseScrub || isFastSeeking) {
|
|
500
|
+
try {
|
|
501
|
+
// Use delayed loading instead of immediate loading
|
|
502
|
+
this.startDelayedLoading(
|
|
503
|
+
"scrub-segment-load",
|
|
504
|
+
"Loading scrub segment...",
|
|
505
|
+
);
|
|
506
|
+
|
|
507
|
+
const scrubFrame =
|
|
508
|
+
await this.scrubTrackManager.getScrubFrame(currentSeekToMs);
|
|
509
|
+
|
|
510
|
+
if (scrubFrame && this.canvasElement) {
|
|
511
|
+
this.scrubTrackManager.recordCacheMiss();
|
|
512
|
+
this.lastSeekTimeMs = currentSeekToMs;
|
|
513
|
+
|
|
514
|
+
// Clear loading and display scrub frame
|
|
515
|
+
this.clearDelayedLoading("scrub-segment-load");
|
|
516
|
+
return this.displayFrame(scrubFrame, currentSeekToMs);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// Scrub frame was null/failed - fall back to normal video
|
|
520
|
+
console.warn(
|
|
521
|
+
"Scrub track returned null frame, falling back to normal video",
|
|
522
|
+
);
|
|
523
|
+
this.clearDelayedLoading("scrub-segment-load");
|
|
524
|
+
this.startDelayedLoading(
|
|
525
|
+
"video-segment-fallback",
|
|
526
|
+
"Loading high quality video...",
|
|
527
|
+
);
|
|
528
|
+
} catch (error) {
|
|
529
|
+
this.clearDelayedLoading("scrub-segment-load");
|
|
530
|
+
console.warn(
|
|
531
|
+
"Scrub track failed, falling back to normal video:",
|
|
532
|
+
error,
|
|
533
|
+
);
|
|
534
|
+
|
|
535
|
+
// Show loading for normal video fallback
|
|
536
|
+
this.startDelayedLoading(
|
|
537
|
+
"video-segment-fallback",
|
|
538
|
+
"Loading high quality video...",
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
} else {
|
|
542
|
+
// Cache hit for normal video - scrub track manager exists, record the hit
|
|
543
|
+
this.scrubTrackManager?.recordCacheHit();
|
|
103
544
|
}
|
|
104
545
|
}
|
|
105
546
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
547
|
+
// Normal video rendering path (for all cases where scrub track isn't used)
|
|
548
|
+
// Check if we need to show loading for normal video operations
|
|
549
|
+
const shouldShowLoading =
|
|
550
|
+
!this.delayedLoadingState.isLoading &&
|
|
551
|
+
(this.effectiveMode !== "asset" || !videoAsset);
|
|
552
|
+
|
|
553
|
+
if (shouldShowLoading) {
|
|
554
|
+
this.startDelayedLoading("video-segment", "Loading video segment...");
|
|
109
555
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
556
|
+
|
|
557
|
+
// Render normal video - pass fresh VideoAsset reference
|
|
558
|
+
this.lastSeekTimeMs = currentSeekToMs;
|
|
559
|
+
const result = await this.renderNormalVideo(
|
|
560
|
+
videoAsset,
|
|
561
|
+
currentSeekToMs,
|
|
116
562
|
);
|
|
117
563
|
|
|
118
|
-
|
|
564
|
+
// Clear loading state after normal video renders
|
|
565
|
+
this.clearDelayedLoading("video-segment");
|
|
566
|
+
this.clearDelayedLoading("video-segment-fallback");
|
|
567
|
+
|
|
568
|
+
return result;
|
|
119
569
|
} catch (error) {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
570
|
+
// Clear all loading states on error
|
|
571
|
+
this.clearDelayedLoading("scrub-segment-load");
|
|
572
|
+
this.clearDelayedLoading("video-segment");
|
|
573
|
+
this.clearDelayedLoading("video-segment-fallback");
|
|
574
|
+
|
|
575
|
+
// Handle errors with proper error propagation
|
|
576
|
+
if (error instanceof Error) {
|
|
577
|
+
if (
|
|
578
|
+
error.name === "DataError" &&
|
|
579
|
+
error.message.includes("key frame is required")
|
|
580
|
+
) {
|
|
581
|
+
console.warn(
|
|
582
|
+
"Decoder reset during VideoAsset due to key frame requirement",
|
|
583
|
+
);
|
|
584
|
+
this.#decoderNeedsReset = true;
|
|
585
|
+
|
|
586
|
+
if (this.effectiveMode === "jit-transcode") {
|
|
587
|
+
this.requestUpdate();
|
|
588
|
+
}
|
|
589
|
+
throw error;
|
|
590
|
+
}
|
|
591
|
+
if (error.name === "AbortError") {
|
|
592
|
+
// AbortError is expected behavior when tasks are cancelled
|
|
593
|
+
throw new Error(
|
|
594
|
+
"Frame rendering cancelled: Operation was aborted, likely due to a new seek request or component unmounting.",
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
if (
|
|
598
|
+
error.message.includes("VideoAsset decoder closed") ||
|
|
599
|
+
error.message.includes("recreation in progress")
|
|
600
|
+
) {
|
|
601
|
+
// This is now expected behavior during VideoAsset transitions - don't treat as error
|
|
602
|
+
return; // Gracefully abort instead of throwing
|
|
603
|
+
}
|
|
604
|
+
if (
|
|
605
|
+
error.name === "InvalidStateError" &&
|
|
606
|
+
error.message.includes("closed codec")
|
|
607
|
+
) {
|
|
608
|
+
// Expected during VideoAsset recreation - gracefully abort
|
|
609
|
+
return; // Gracefully abort instead of throwing
|
|
610
|
+
}
|
|
611
|
+
console.warn("Decoder reset during VideoAsset recreation", error);
|
|
612
|
+
this.#decoderNeedsReset = true;
|
|
613
|
+
throw error;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// For non-Error objects, still provide descriptive error
|
|
617
|
+
throw new Error(
|
|
618
|
+
`Frame rendering failed: Unknown error during video rendering at ${currentSeekToMs}ms. Error: ${String(error)}`,
|
|
619
|
+
);
|
|
123
620
|
} finally {
|
|
124
621
|
this.#decoderLock = false;
|
|
125
622
|
}
|
|
126
623
|
},
|
|
127
624
|
});
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Render normal video using existing logic
|
|
628
|
+
*/
|
|
629
|
+
private async renderNormalVideo(
|
|
630
|
+
videoAsset: VideoAsset,
|
|
631
|
+
seekToMs: number,
|
|
632
|
+
): Promise<number> {
|
|
633
|
+
let targetSeekTimeSeconds = seekToMs / 1000;
|
|
634
|
+
|
|
635
|
+
try {
|
|
636
|
+
// Validate VideoAsset is still current before seeking
|
|
637
|
+
const currentVideoAsset = this.videoAssetTask.value;
|
|
638
|
+
if (videoAsset !== currentVideoAsset) {
|
|
639
|
+
throw new Error(
|
|
640
|
+
"VideoAsset decoder closed during seek - recreation in progress",
|
|
641
|
+
);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// Check decoder state immediately before seeking
|
|
645
|
+
const decoderState = videoAsset?.videoDecoder?.state;
|
|
646
|
+
if (decoderState === "closed") {
|
|
647
|
+
throw new Error(
|
|
648
|
+
"VideoAsset decoder closed during seek - recreation in progress",
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
if (this.effectiveMode === "jit-transcode") {
|
|
652
|
+
targetSeekTimeSeconds %= 2;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
const frame = await videoAsset.seekToTime(targetSeekTimeSeconds);
|
|
656
|
+
|
|
657
|
+
if (frame) {
|
|
658
|
+
// Final validation that VideoAsset is still current before displaying
|
|
659
|
+
const finalVideoAsset = this.videoAssetTask.value;
|
|
660
|
+
if (videoAsset !== finalVideoAsset) {
|
|
661
|
+
frame.close(); // Clean up the frame
|
|
662
|
+
throw new Error(
|
|
663
|
+
"VideoAsset decoder closed during seek - recreation in progress",
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
// Read fresh time right before displaying - final safeguard against stale values
|
|
668
|
+
const finalSeekToMs = this.desiredSeekTimeMs;
|
|
669
|
+
return this.displayFrame(frame, finalSeekToMs);
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
log("trace: no frame returned from seekToTime");
|
|
673
|
+
throw new Error(
|
|
674
|
+
`Frame rendering failed: No frame available at time ${seekToMs}ms (${targetSeekTimeSeconds}s). This may indicate seeking beyond video duration, corrupted video data, or an incompatible video format.`,
|
|
675
|
+
);
|
|
676
|
+
} catch (error) {
|
|
677
|
+
if (
|
|
678
|
+
error instanceof Error &&
|
|
679
|
+
(error.message.includes("VideoAsset decoder closed") ||
|
|
680
|
+
error.message.includes("recreation in progress"))
|
|
681
|
+
) {
|
|
682
|
+
// This is the expected narrow timing window race condition during VideoAsset transitions
|
|
683
|
+
throw error; // Re-throw to let paintTask handle it gracefully
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// Re-throw other unexpected errors
|
|
687
|
+
throw error;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Display a video frame on the canvas
|
|
693
|
+
*/
|
|
694
|
+
private displayFrame(frame: VideoFrame, seekToMs: number): number {
|
|
695
|
+
log("trace: displayFrame start", { seekToMs, frameFormat: frame.format });
|
|
696
|
+
if (!this.canvasElement) {
|
|
697
|
+
log("trace: displayFrame aborted - no canvas element");
|
|
698
|
+
throw new Error(
|
|
699
|
+
`Frame display failed: Canvas element is not available at time ${seekToMs}ms. The video component may not be properly initialized.`,
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
const ctx = this.canvasElement.getContext("2d");
|
|
704
|
+
if (!ctx) {
|
|
705
|
+
log("trace: displayFrame aborted - no canvas context");
|
|
706
|
+
throw new Error(
|
|
707
|
+
`Frame display failed: Unable to get 2D canvas context at time ${seekToMs}ms. This may indicate a browser compatibility issue or canvas corruption.`,
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
if (frame?.codedWidth && frame?.codedHeight) {
|
|
712
|
+
if (
|
|
713
|
+
this.canvasElement.width !== frame.codedWidth ||
|
|
714
|
+
this.canvasElement.height !== frame.codedHeight
|
|
715
|
+
) {
|
|
716
|
+
log("trace: updating canvas dimensions", {
|
|
717
|
+
width: frame.codedWidth,
|
|
718
|
+
height: frame.codedHeight,
|
|
719
|
+
});
|
|
720
|
+
this.canvasElement.width = frame.codedWidth;
|
|
721
|
+
this.canvasElement.height = frame.codedHeight;
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
if (frame.format === null) {
|
|
726
|
+
log("trace: displayFrame aborted - null frame format");
|
|
727
|
+
throw new Error(
|
|
728
|
+
`Frame display failed: Video frame has null format at time ${seekToMs}ms. This indicates corrupted or incompatible video data.`,
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
log("trace: drawing frame to canvas");
|
|
733
|
+
ctx.drawImage(
|
|
734
|
+
frame,
|
|
735
|
+
0,
|
|
736
|
+
0,
|
|
737
|
+
this.canvasElement.width,
|
|
738
|
+
this.canvasElement.height,
|
|
739
|
+
);
|
|
740
|
+
log("trace: frame drawn to canvas", { seekToMs });
|
|
741
|
+
|
|
742
|
+
return seekToMs;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Check if we're in production rendering mode (EF_FRAMEGEN active) vs preview mode
|
|
747
|
+
*/
|
|
748
|
+
private isInProductionRenderingMode(): boolean {
|
|
749
|
+
// Check if EF_RENDERING function exists and returns true (production rendering)
|
|
750
|
+
if (typeof window.EF_RENDERING === "function") {
|
|
751
|
+
return window.EF_RENDERING();
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// Check if workbench is in rendering mode
|
|
755
|
+
const workbench = document.querySelector("ef-workbench") as any;
|
|
756
|
+
if (workbench?.rendering) {
|
|
757
|
+
return true;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// Check if EF_FRAMEGEN exists and has render options (indicates active rendering)
|
|
761
|
+
if (window.EF_FRAMEGEN?.renderOptions) {
|
|
762
|
+
return true;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
// Default to preview mode
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* Check if EF_FRAMEGEN has explicitly started frame rendering (not just initialization)
|
|
771
|
+
*/
|
|
772
|
+
private isFrameRenderingActive(): boolean {
|
|
773
|
+
if (!window.EF_FRAMEGEN?.renderOptions) {
|
|
774
|
+
return false;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// In production mode, only render when EF_FRAMEGEN has actually begun frame sequence
|
|
778
|
+
// Check if we're past the initialization phase by looking for explicit frame control
|
|
779
|
+
const renderOptions = window.EF_FRAMEGEN.renderOptions;
|
|
780
|
+
const renderStartTime = renderOptions.encoderOptions.fromMs;
|
|
781
|
+
const currentTime = this.rootTimegroup?.currentTimeMs || 0;
|
|
782
|
+
|
|
783
|
+
// We're in active frame rendering if:
|
|
784
|
+
// 1. currentTime >= renderStartTime (includes the starting frame)
|
|
785
|
+
return currentTime >= renderStartTime;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Get scrub track performance statistics
|
|
790
|
+
*/
|
|
791
|
+
getScrubTrackStats(): CacheStats | null {
|
|
792
|
+
return this.scrubTrackManager?.getCacheStats() || null;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/**
|
|
796
|
+
* Clean up resources when component is disconnected
|
|
797
|
+
*/
|
|
798
|
+
disconnectedCallback(): void {
|
|
799
|
+
super.disconnectedCallback();
|
|
800
|
+
|
|
801
|
+
// Clean up scrub track manager
|
|
802
|
+
this.scrubTrackManager?.cleanup();
|
|
803
|
+
|
|
804
|
+
// Clean up delayed loading state
|
|
805
|
+
this.delayedLoadingState.clearAllLoading();
|
|
806
|
+
}
|
|
128
807
|
}
|
|
129
808
|
|
|
130
809
|
declare global {
|