@editframe/elements 0.26.2-beta.0 → 0.26.4-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/elements/EFTimegroup.js +7 -2
- package/dist/elements/EFTimegroup.js.map +1 -1
- package/package.json +2 -2
- package/scripts/build-css.js +3 -3
- package/tsdown.config.ts +1 -1
- package/types.json +1 -1
- package/src/elements/ContextProxiesController.ts +0 -124
- package/src/elements/CrossUpdateController.ts +0 -22
- package/src/elements/EFAudio.browsertest.ts +0 -706
- package/src/elements/EFAudio.ts +0 -56
- package/src/elements/EFCaptions.browsertest.ts +0 -1960
- package/src/elements/EFCaptions.ts +0 -823
- package/src/elements/EFImage.browsertest.ts +0 -120
- package/src/elements/EFImage.ts +0 -113
- package/src/elements/EFMedia/AssetIdMediaEngine.test.ts +0 -224
- package/src/elements/EFMedia/AssetIdMediaEngine.ts +0 -110
- package/src/elements/EFMedia/AssetMediaEngine.browsertest.ts +0 -140
- package/src/elements/EFMedia/AssetMediaEngine.ts +0 -385
- package/src/elements/EFMedia/BaseMediaEngine.browsertest.ts +0 -400
- package/src/elements/EFMedia/BaseMediaEngine.ts +0 -505
- package/src/elements/EFMedia/BufferedSeekingInput.browsertest.ts +0 -386
- package/src/elements/EFMedia/BufferedSeekingInput.ts +0 -430
- package/src/elements/EFMedia/JitMediaEngine.browsertest.ts +0 -226
- package/src/elements/EFMedia/JitMediaEngine.ts +0 -256
- package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.browsertest.ts +0 -679
- package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.ts +0 -117
- package/src/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts +0 -246
- package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.browsertest.ts +0 -59
- package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts +0 -27
- package/src/elements/EFMedia/audioTasks/makeAudioInputTask.browsertest.ts +0 -55
- package/src/elements/EFMedia/audioTasks/makeAudioInputTask.ts +0 -53
- package/src/elements/EFMedia/audioTasks/makeAudioSeekTask.chunkboundary.regression.browsertest.ts +0 -207
- package/src/elements/EFMedia/audioTasks/makeAudioSeekTask.ts +0 -72
- package/src/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.ts +0 -32
- package/src/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.ts +0 -29
- package/src/elements/EFMedia/audioTasks/makeAudioTasksVideoOnly.browsertest.ts +0 -95
- package/src/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts +0 -184
- package/src/elements/EFMedia/shared/AudioSpanUtils.ts +0 -129
- package/src/elements/EFMedia/shared/BufferUtils.ts +0 -342
- package/src/elements/EFMedia/shared/GlobalInputCache.ts +0 -77
- package/src/elements/EFMedia/shared/MediaTaskUtils.ts +0 -44
- package/src/elements/EFMedia/shared/PrecisionUtils.ts +0 -46
- package/src/elements/EFMedia/shared/RenditionHelpers.browsertest.ts +0 -246
- package/src/elements/EFMedia/shared/RenditionHelpers.ts +0 -56
- package/src/elements/EFMedia/shared/ThumbnailExtractor.ts +0 -227
- package/src/elements/EFMedia/tasks/makeMediaEngineTask.browsertest.ts +0 -167
- package/src/elements/EFMedia/tasks/makeMediaEngineTask.ts +0 -88
- package/src/elements/EFMedia/videoTasks/MainVideoInputCache.ts +0 -76
- package/src/elements/EFMedia/videoTasks/ScrubInputCache.ts +0 -61
- package/src/elements/EFMedia/videoTasks/makeScrubVideoBufferTask.ts +0 -114
- package/src/elements/EFMedia/videoTasks/makeScrubVideoInitSegmentFetchTask.ts +0 -35
- package/src/elements/EFMedia/videoTasks/makeScrubVideoInputTask.ts +0 -52
- package/src/elements/EFMedia/videoTasks/makeScrubVideoSeekTask.ts +0 -124
- package/src/elements/EFMedia/videoTasks/makeScrubVideoSegmentFetchTask.ts +0 -44
- package/src/elements/EFMedia/videoTasks/makeScrubVideoSegmentIdTask.ts +0 -32
- package/src/elements/EFMedia/videoTasks/makeUnifiedVideoSeekTask.ts +0 -370
- package/src/elements/EFMedia/videoTasks/makeVideoBufferTask.ts +0 -109
- package/src/elements/EFMedia.browsertest.ts +0 -872
- package/src/elements/EFMedia.ts +0 -341
- package/src/elements/EFSourceMixin.ts +0 -60
- package/src/elements/EFSurface.browsertest.ts +0 -151
- package/src/elements/EFSurface.ts +0 -142
- package/src/elements/EFTemporal.browsertest.ts +0 -215
- package/src/elements/EFTemporal.ts +0 -800
- package/src/elements/EFThumbnailStrip.browsertest.ts +0 -585
- package/src/elements/EFThumbnailStrip.media-engine.browsertest.ts +0 -714
- package/src/elements/EFThumbnailStrip.ts +0 -906
- package/src/elements/EFTimegroup.browsertest.ts +0 -870
- package/src/elements/EFTimegroup.ts +0 -878
- package/src/elements/EFVideo.browsertest.ts +0 -1482
- package/src/elements/EFVideo.ts +0 -564
- package/src/elements/EFWaveform.ts +0 -547
- package/src/elements/FetchContext.browsertest.ts +0 -401
- package/src/elements/FetchMixin.ts +0 -38
- package/src/elements/SampleBuffer.ts +0 -94
- package/src/elements/TargetController.browsertest.ts +0 -230
- package/src/elements/TargetController.ts +0 -224
- package/src/elements/TimegroupController.ts +0 -26
- package/src/elements/durationConverter.ts +0 -35
- package/src/elements/parseTimeToMs.ts +0 -9
- package/src/elements/printTaskStatus.ts +0 -16
- package/src/elements/renderTemporalAudio.ts +0 -108
- package/src/elements/updateAnimations.browsertest.ts +0 -1884
- package/src/elements/updateAnimations.ts +0 -217
- package/src/elements/util.ts +0 -24
- package/src/gui/ContextMixin.browsertest.ts +0 -860
- package/src/gui/ContextMixin.ts +0 -562
- package/src/gui/Controllable.browsertest.ts +0 -258
- package/src/gui/Controllable.ts +0 -41
- package/src/gui/EFConfiguration.ts +0 -40
- package/src/gui/EFControls.browsertest.ts +0 -389
- package/src/gui/EFControls.ts +0 -195
- package/src/gui/EFDial.browsertest.ts +0 -84
- package/src/gui/EFDial.ts +0 -172
- package/src/gui/EFFilmstrip.browsertest.ts +0 -712
- package/src/gui/EFFilmstrip.ts +0 -1349
- package/src/gui/EFFitScale.ts +0 -152
- package/src/gui/EFFocusOverlay.ts +0 -79
- package/src/gui/EFPause.browsertest.ts +0 -202
- package/src/gui/EFPause.ts +0 -73
- package/src/gui/EFPlay.browsertest.ts +0 -202
- package/src/gui/EFPlay.ts +0 -73
- package/src/gui/EFPreview.ts +0 -74
- package/src/gui/EFResizableBox.browsertest.ts +0 -79
- package/src/gui/EFResizableBox.ts +0 -898
- package/src/gui/EFScrubber.ts +0 -151
- package/src/gui/EFTimeDisplay.browsertest.ts +0 -237
- package/src/gui/EFTimeDisplay.ts +0 -55
- package/src/gui/EFToggleLoop.ts +0 -35
- package/src/gui/EFTogglePlay.ts +0 -70
- package/src/gui/EFWorkbench.ts +0 -115
- package/src/gui/PlaybackController.ts +0 -527
- package/src/gui/TWMixin.css +0 -6
- package/src/gui/TWMixin.ts +0 -61
- package/src/gui/TargetOrContextMixin.ts +0 -185
- package/src/gui/currentTimeContext.ts +0 -5
- package/src/gui/durationContext.ts +0 -3
- package/src/gui/efContext.ts +0 -6
- package/src/gui/fetchContext.ts +0 -5
- package/src/gui/focusContext.ts +0 -7
- package/src/gui/focusedElementContext.ts +0 -5
- package/src/gui/playingContext.ts +0 -5
- package/src/otel/BridgeSpanExporter.ts +0 -150
- package/src/otel/setupBrowserTracing.ts +0 -73
- package/src/otel/tracingHelpers.ts +0 -251
- package/src/transcoding/cache/RequestDeduplicator.test.ts +0 -170
- package/src/transcoding/cache/RequestDeduplicator.ts +0 -65
- package/src/transcoding/cache/URLTokenDeduplicator.test.ts +0 -182
- package/src/transcoding/cache/URLTokenDeduplicator.ts +0 -101
- package/src/transcoding/types/index.ts +0 -312
- package/src/transcoding/utils/MediaUtils.ts +0 -63
- package/src/transcoding/utils/UrlGenerator.ts +0 -68
- package/src/transcoding/utils/constants.ts +0 -36
- package/src/utils/LRUCache.test.ts +0 -274
- package/src/utils/LRUCache.ts +0 -696
package/src/elements/EFMedia/audioTasks/makeAudioSeekTask.chunkboundary.regression.browsertest.ts
DELETED
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
import { afterEach, beforeEach, describe } from "vitest";
|
|
2
|
-
import { test as baseTest } from "../../../../test/useMSW.js";
|
|
3
|
-
import type { EFConfiguration } from "../../../gui/EFConfiguration.js";
|
|
4
|
-
import "../../../gui/EFPreview.js";
|
|
5
|
-
import "../../EFTimegroup.js";
|
|
6
|
-
import type { EFTimegroup } from "../../EFTimegroup.js";
|
|
7
|
-
import "../../EFVideo.js";
|
|
8
|
-
import type { EFVideo } from "../../EFVideo.js";
|
|
9
|
-
|
|
10
|
-
const test = baseTest.extend<{
|
|
11
|
-
timegroup: EFTimegroup;
|
|
12
|
-
video: EFVideo;
|
|
13
|
-
configuration: EFConfiguration;
|
|
14
|
-
}>({
|
|
15
|
-
timegroup: async ({}, use) => {
|
|
16
|
-
const timegroup = document.createElement("ef-timegroup");
|
|
17
|
-
timegroup.setAttribute("mode", "sequence");
|
|
18
|
-
timegroup.setAttribute("id", "test-timegroup"); // Required for localStorage key
|
|
19
|
-
timegroup.style.cssText =
|
|
20
|
-
"position: relative; height: 500px; width: 1000px; overflow: hidden; background-color: rgb(100 116 139);";
|
|
21
|
-
await use(timegroup);
|
|
22
|
-
},
|
|
23
|
-
configuration: async ({ expect }, use) => {
|
|
24
|
-
const configuration = document.createElement("ef-configuration");
|
|
25
|
-
configuration.innerHTML = `<h1 style="font: 10px monospace">${expect.getState().currentTestName}</h1>`;
|
|
26
|
-
// Use integrated proxy server (same host/port as test runner)
|
|
27
|
-
const apiHost = `${window.location.protocol}//${window.location.host}`;
|
|
28
|
-
configuration.setAttribute("api-host", apiHost);
|
|
29
|
-
configuration.apiHost = apiHost;
|
|
30
|
-
document.body.appendChild(configuration);
|
|
31
|
-
await use(configuration);
|
|
32
|
-
},
|
|
33
|
-
video: async ({ configuration, timegroup }, use) => {
|
|
34
|
-
const video = document.createElement("ef-video");
|
|
35
|
-
video.id = "bars-n-tone2";
|
|
36
|
-
video.src = "http://web:3000/head-moov-480p.mp4"; // Real video from working simple-demo
|
|
37
|
-
video.style.cssText =
|
|
38
|
-
"width: 100%; height: 100%; object-fit: cover; position: absolute; top: 0; left: 0;";
|
|
39
|
-
|
|
40
|
-
// Create the exact structure from simple-demo.html
|
|
41
|
-
const innerTimegroup = document.createElement("ef-timegroup");
|
|
42
|
-
innerTimegroup.mode = "contain";
|
|
43
|
-
innerTimegroup.style.cssText =
|
|
44
|
-
"position: absolute; width: 100%; height: 100%;";
|
|
45
|
-
innerTimegroup.append(video);
|
|
46
|
-
timegroup.append(innerTimegroup);
|
|
47
|
-
configuration.append(timegroup);
|
|
48
|
-
|
|
49
|
-
await use(video);
|
|
50
|
-
},
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Regression test for chunk boundary seeking issue
|
|
55
|
-
*
|
|
56
|
-
* Root cause: 32ms coordination gap between PlaybackController and audio track boundaries
|
|
57
|
-
* - PlaybackController seeks to chunk boundary: 4000ms
|
|
58
|
-
* - Audio track actually starts at: 4032ms
|
|
59
|
-
* - Error: "Seek time 4000ms is outside track range [4032ms, 6016ms]"
|
|
60
|
-
*
|
|
61
|
-
* This occurs during active playbook and browser reloads at 4s mark.
|
|
62
|
-
* Fix: Coordinate chunk boundaries or add tolerance for small gaps.
|
|
63
|
-
*/
|
|
64
|
-
describe("Audio Seek Task - Chunk Boundary Regression Test", () => {
|
|
65
|
-
beforeEach(() => {
|
|
66
|
-
// Clean up DOM and localStorage
|
|
67
|
-
while (document.body.children.length) {
|
|
68
|
-
document.body.children[0]?.remove();
|
|
69
|
-
}
|
|
70
|
-
localStorage.clear();
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
afterEach(async () => {
|
|
74
|
-
// Clean up any remaining elements
|
|
75
|
-
const videos = document.querySelectorAll("ef-video");
|
|
76
|
-
for (const video of videos) {
|
|
77
|
-
video.remove();
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
test.skip("should not throw RangeError when seeking to exact 4000ms during playback", async ({
|
|
82
|
-
// SKIP: audioSeekTask is not part of the audio rendering pipeline
|
|
83
|
-
video,
|
|
84
|
-
timegroup,
|
|
85
|
-
expect,
|
|
86
|
-
}) => {
|
|
87
|
-
await video.mediaEngineTask.taskComplete;
|
|
88
|
-
await video.audioInputTask.taskComplete;
|
|
89
|
-
|
|
90
|
-
// Simulate active playback - start playing from beginning
|
|
91
|
-
timegroup.currentTimeMs = 0;
|
|
92
|
-
await video.audioSeekTask.taskComplete;
|
|
93
|
-
|
|
94
|
-
// Now seek to the exact problematic time that causes:
|
|
95
|
-
// "Seek time 4000ms is outside track range [4032ms, 6016ms]"
|
|
96
|
-
const exactChunkBoundary = 4000;
|
|
97
|
-
timegroup.currentTimeMs = exactChunkBoundary;
|
|
98
|
-
|
|
99
|
-
// Should not throw RangeError due to track range mismatch
|
|
100
|
-
await expect(video.audioSeekTask.taskComplete).resolves.toBeDefined();
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
test.skip("should not throw RangeError during progressive playback across segments", async ({
|
|
104
|
-
// SKIP: audioSeekTask is not part of the audio rendering pipeline
|
|
105
|
-
video,
|
|
106
|
-
timegroup,
|
|
107
|
-
expect,
|
|
108
|
-
}) => {
|
|
109
|
-
await video.mediaEngineTask.taskComplete;
|
|
110
|
-
await video.audioInputTask.taskComplete;
|
|
111
|
-
|
|
112
|
-
// Simulate progressive playback that loads segments on demand
|
|
113
|
-
// Start at 3500ms to be just before the 4-second boundary
|
|
114
|
-
timegroup.currentTimeMs = 3500;
|
|
115
|
-
await video.audioSeekTask.taskComplete;
|
|
116
|
-
|
|
117
|
-
// Now cross the 4-second chunk boundary where track range issues occur
|
|
118
|
-
// This should trigger the state where track range is [4032ms, 6016ms]
|
|
119
|
-
// but we're seeking to 4000ms
|
|
120
|
-
timegroup.currentTimeMs = 4000.000000000001; // The exact error from logs
|
|
121
|
-
|
|
122
|
-
// Should not throw "Seek time 4000.000000000001ms is outside track range [4032ms, 6016ms]"
|
|
123
|
-
await expect(video.audioSeekTask.taskComplete).resolves.toBeDefined();
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
test.skip("should not throw RangeError when localStorage restoration causes 0ms to 4000ms race condition", async ({
|
|
127
|
-
// SKIP: audioSeekTask is not part of the audio rendering pipeline
|
|
128
|
-
video,
|
|
129
|
-
timegroup,
|
|
130
|
-
expect,
|
|
131
|
-
}) => {
|
|
132
|
-
// REPRODUCE THE RACE CONDITION: Simulate localStorage having "4.0"
|
|
133
|
-
// This mimics the exact simple-demo.html scenario where:
|
|
134
|
-
// 1. Media loads with assumption of currentTimeMs = 0
|
|
135
|
-
// 2. localStorage restores currentTime to 4.0 seconds
|
|
136
|
-
// 3. Seeking 4000ms in segments loaded for 0ms range triggers RangeError
|
|
137
|
-
|
|
138
|
-
// Set localStorage BEFORE media finishes initializing
|
|
139
|
-
if (timegroup.id) {
|
|
140
|
-
localStorage.setItem(`ef-timegroup-${timegroup.id}`, "4.0");
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// Wait for media engine but NOT for full initialization
|
|
144
|
-
await video.mediaEngineTask.taskComplete;
|
|
145
|
-
|
|
146
|
-
// Now trigger the localStorage restoration that happens in waitForMediaDurations().then()
|
|
147
|
-
// This will load currentTime = 4.0 from localStorage, jumping from 0ms to 4000ms
|
|
148
|
-
const loadedTime = timegroup.loadTimeFromLocalStorage();
|
|
149
|
-
if (loadedTime !== undefined) {
|
|
150
|
-
timegroup.currentTime = loadedTime;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// This should trigger: "Seek time 4000ms is outside track range [Yms, Zms]"
|
|
154
|
-
// because segments were loaded for 0ms but we're now seeking 4000ms
|
|
155
|
-
await expect(video.audioSeekTask.taskComplete).resolves.toBeDefined();
|
|
156
|
-
});
|
|
157
|
-
|
|
158
|
-
test.skip("should not throw RangeError when forced segment coordination mismatch occurs", async ({
|
|
159
|
-
// SKIP: audioSeekTask is not part of the audio rendering pipeline
|
|
160
|
-
video,
|
|
161
|
-
timegroup,
|
|
162
|
-
expect,
|
|
163
|
-
}) => {
|
|
164
|
-
await video.mediaEngineTask.taskComplete;
|
|
165
|
-
|
|
166
|
-
// FORCE SPECIFIC SEGMENT LOADING: Load a segment for 8000ms (segment 5)
|
|
167
|
-
timegroup.currentTimeMs = 8000;
|
|
168
|
-
await video.audioSegmentIdTask.taskComplete;
|
|
169
|
-
await video.audioSegmentFetchTask.taskComplete;
|
|
170
|
-
await video.audioInputTask.taskComplete;
|
|
171
|
-
|
|
172
|
-
// Verify we have segment 5 loaded (8000ms / 15000ms = segment 1, but 1-based = segment 1...
|
|
173
|
-
// Actually 8000ms maps to segment 5 based on the actual segment calculation)
|
|
174
|
-
const segmentId = video.audioSegmentIdTask.value;
|
|
175
|
-
expect(segmentId).toBe(4);
|
|
176
|
-
|
|
177
|
-
// Now seek to a time in a different segment to test coordination
|
|
178
|
-
timegroup.currentTimeMs = 4000;
|
|
179
|
-
|
|
180
|
-
// This tests the fundamental segment coordination issue:
|
|
181
|
-
// - We loaded segment 5 for 8000ms
|
|
182
|
-
// - Now seeking to 4000ms which should be in a different segment
|
|
183
|
-
// - Tests that seek doesn't fail due to segment boundary coordination
|
|
184
|
-
await expect(video.audioSeekTask.taskComplete).resolves.toBeDefined();
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
test.skip("should not throw RangeError when rapidly crossing segment boundaries", async ({
|
|
188
|
-
// SKIP: audioSeekTask is not part of the audio rendering pipeline
|
|
189
|
-
video,
|
|
190
|
-
timegroup,
|
|
191
|
-
expect,
|
|
192
|
-
}) => {
|
|
193
|
-
await video.mediaEngineTask.taskComplete;
|
|
194
|
-
|
|
195
|
-
// RAPID BOUNDARY CROSSING: This tests timing-sensitive segment coordination
|
|
196
|
-
const boundaries = [1000, 4000, 8000, 3000, 7000]; // Jump around within segment 1
|
|
197
|
-
|
|
198
|
-
for (const timeMs of boundaries) {
|
|
199
|
-
timegroup.currentTimeMs = timeMs;
|
|
200
|
-
// Don't await - test rapid succession to trigger coordination issues
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
// Final seek - this should not throw even after rapid boundary crossing
|
|
204
|
-
timegroup.currentTimeMs = 4000;
|
|
205
|
-
await expect(video.audioSeekTask.taskComplete).resolves.toBeDefined();
|
|
206
|
-
});
|
|
207
|
-
});
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import { Task } from "@lit/task";
|
|
2
|
-
import type { VideoSample } from "mediabunny";
|
|
3
|
-
import { type EFMedia, IgnorableError } from "../../EFMedia";
|
|
4
|
-
import type { BufferedSeekingInput } from "../BufferedSeekingInput";
|
|
5
|
-
|
|
6
|
-
type AudioSeekTask = Task<
|
|
7
|
-
readonly [number, BufferedSeekingInput | undefined],
|
|
8
|
-
VideoSample | undefined
|
|
9
|
-
>;
|
|
10
|
-
export const makeAudioSeekTask = (host: EFMedia): AudioSeekTask => {
|
|
11
|
-
return new Task(host, {
|
|
12
|
-
args: () => [host.desiredSeekTimeMs, host.audioInputTask.value] as const,
|
|
13
|
-
onError: (error) => {
|
|
14
|
-
if (error instanceof IgnorableError) {
|
|
15
|
-
console.info("audioSeekTask aborted");
|
|
16
|
-
return;
|
|
17
|
-
}
|
|
18
|
-
if (error instanceof DOMException) {
|
|
19
|
-
console.error(
|
|
20
|
-
`audioSeekTask error: ${error.message} ${error.name} ${error.code}`,
|
|
21
|
-
);
|
|
22
|
-
} else if (error instanceof Error) {
|
|
23
|
-
console.error(`audioSeekTask error ${error.name}: ${error.message}`);
|
|
24
|
-
} else {
|
|
25
|
-
console.error("audioSeekTask unknown error", error);
|
|
26
|
-
}
|
|
27
|
-
},
|
|
28
|
-
onComplete: (_value) => {},
|
|
29
|
-
task: async (): Promise<VideoSample | undefined> => {
|
|
30
|
-
return undefined;
|
|
31
|
-
// TODO: validate that the audio seek task is not actually used to render any audio
|
|
32
|
-
// CRITICAL FIX: Use the targetSeekTimeMs from args, not host.desiredSeekTimeMs
|
|
33
|
-
// This ensures we use the same seek time that the segment loading tasks used
|
|
34
|
-
|
|
35
|
-
// await host.audioSegmentIdTask.taskComplete;
|
|
36
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
37
|
-
// await host.audioSegmentFetchTask.taskComplete;
|
|
38
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
39
|
-
// await host.audioInitSegmentFetchTask.taskComplete;
|
|
40
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
41
|
-
|
|
42
|
-
// const audioInput = await host.audioInputTask.taskComplete;
|
|
43
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
44
|
-
// if (!audioInput) {
|
|
45
|
-
// throw new Error("Audio input is not available");
|
|
46
|
-
// }
|
|
47
|
-
// const audioTrack = await audioInput.getFirstAudioTrack();
|
|
48
|
-
// if (!audioTrack) {
|
|
49
|
-
// throw new Error("Audio track is not available");
|
|
50
|
-
// }
|
|
51
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
52
|
-
|
|
53
|
-
// const sample = (await audioInput.seek(
|
|
54
|
-
// audioTrack.id,
|
|
55
|
-
// targetSeekTimeMs, // Use the captured value, not host.desiredSeekTimeMs
|
|
56
|
-
// )) as unknown as VideoSample | undefined;
|
|
57
|
-
// signal.throwIfAborted(); // Abort if a new seek started
|
|
58
|
-
|
|
59
|
-
// // If seek returned undefined, it was aborted - don't throw
|
|
60
|
-
// if (sample === undefined && signal.aborted) {
|
|
61
|
-
// return undefined;
|
|
62
|
-
// }
|
|
63
|
-
|
|
64
|
-
// // If we got undefined but weren't aborted, that's an actual error
|
|
65
|
-
// if (sample === undefined) {
|
|
66
|
-
// throw new Error("Audio seek failed to find sample");
|
|
67
|
-
// }
|
|
68
|
-
|
|
69
|
-
// return sample;
|
|
70
|
-
},
|
|
71
|
-
});
|
|
72
|
-
};
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { Task } from "@lit/task";
|
|
2
|
-
import type { MediaEngine } from "../../../transcoding/types";
|
|
3
|
-
import type { EFMedia } from "../../EFMedia";
|
|
4
|
-
import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
|
|
5
|
-
|
|
6
|
-
export const makeAudioSegmentFetchTask = (
|
|
7
|
-
host: EFMedia,
|
|
8
|
-
): Task<
|
|
9
|
-
readonly [MediaEngine | undefined, number | undefined],
|
|
10
|
-
ArrayBuffer | undefined
|
|
11
|
-
> => {
|
|
12
|
-
return new Task(host, {
|
|
13
|
-
args: () =>
|
|
14
|
-
[host.mediaEngineTask.value, host.audioSegmentIdTask.value] as const,
|
|
15
|
-
onError: (error) => {
|
|
16
|
-
console.error("audioSegmentFetchTask error", error);
|
|
17
|
-
},
|
|
18
|
-
onComplete: (_value) => {},
|
|
19
|
-
task: async (_, { signal }) => {
|
|
20
|
-
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
21
|
-
const segmentId = await host.audioSegmentIdTask.taskComplete;
|
|
22
|
-
const audioRendition = mediaEngine.getAudioRendition();
|
|
23
|
-
|
|
24
|
-
// Return undefined if no audio rendition or segment ID available (video-only asset)
|
|
25
|
-
if (!audioRendition || segmentId === undefined) {
|
|
26
|
-
return undefined;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
return mediaEngine.fetchMediaSegment(segmentId, audioRendition, signal);
|
|
30
|
-
},
|
|
31
|
-
});
|
|
32
|
-
};
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import { Task } from "@lit/task";
|
|
2
|
-
import type { MediaEngine } from "../../../transcoding/types";
|
|
3
|
-
import type { EFMedia } from "../../EFMedia";
|
|
4
|
-
import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
|
|
5
|
-
|
|
6
|
-
export const makeAudioSegmentIdTask = (
|
|
7
|
-
host: EFMedia,
|
|
8
|
-
): Task<readonly [MediaEngine | undefined, number], number | undefined> => {
|
|
9
|
-
return new Task(host, {
|
|
10
|
-
args: () => [host.mediaEngineTask.value, host.desiredSeekTimeMs] as const,
|
|
11
|
-
onError: (error) => {
|
|
12
|
-
console.error("audioSegmentIdTask error", error);
|
|
13
|
-
},
|
|
14
|
-
onComplete: (_value) => {},
|
|
15
|
-
task: async ([, targetSeekTimeMs], { signal }) => {
|
|
16
|
-
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
17
|
-
signal.throwIfAborted();
|
|
18
|
-
|
|
19
|
-
const audioRendition = mediaEngine.getAudioRendition();
|
|
20
|
-
|
|
21
|
-
// Return undefined if no audio rendition available (video-only asset)
|
|
22
|
-
if (!audioRendition) {
|
|
23
|
-
return undefined;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
return mediaEngine.computeSegmentId(targetSeekTimeMs, audioRendition);
|
|
27
|
-
},
|
|
28
|
-
});
|
|
29
|
-
};
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import { describe } from "vitest";
|
|
2
|
-
import { test as baseTest } from "../../../../test/useMSW.js";
|
|
3
|
-
import type { EFMedia } from "../../EFMedia.js";
|
|
4
|
-
import { AssetMediaEngine } from "../AssetMediaEngine.js";
|
|
5
|
-
|
|
6
|
-
const test = baseTest.extend<{
|
|
7
|
-
videoOnlyAssetEngine: AssetMediaEngine;
|
|
8
|
-
}>({
|
|
9
|
-
videoOnlyAssetEngine: async ({}, use) => {
|
|
10
|
-
const host = document.createElement("ef-video") as EFMedia;
|
|
11
|
-
const engine = new AssetMediaEngine(host, "test-video-only.mp4");
|
|
12
|
-
|
|
13
|
-
// Simulate video-only asset data (no audio track) - this is the exact scenario
|
|
14
|
-
// that caused "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
|
|
15
|
-
(engine as any).data = {
|
|
16
|
-
1: {
|
|
17
|
-
track: 1,
|
|
18
|
-
type: "video",
|
|
19
|
-
width: 480,
|
|
20
|
-
height: 270,
|
|
21
|
-
timescale: 15360,
|
|
22
|
-
sample_count: 1,
|
|
23
|
-
codec: "avc1.640015",
|
|
24
|
-
duration: 30208,
|
|
25
|
-
startTimeOffsetMs: 67,
|
|
26
|
-
initSegment: { offset: 0, size: 763 },
|
|
27
|
-
segments: [
|
|
28
|
-
{ cts: 1024, dts: 0, duration: 30720, offset: 763, size: 13997 },
|
|
29
|
-
],
|
|
30
|
-
},
|
|
31
|
-
// Note: No track 2 (audio) - this simulates the exact video-only asset scenario
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
await use(engine);
|
|
35
|
-
},
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Regression test for: "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
|
|
40
|
-
*
|
|
41
|
-
* This test ensures that AssetMediaEngine properly handles video-only assets
|
|
42
|
-
* by returning undefined for audio renditions instead of malformed objects.
|
|
43
|
-
*
|
|
44
|
-
* This test would FAIL with the old implementation and PASS with the new implementation.
|
|
45
|
-
*/
|
|
46
|
-
describe("AssetMediaEngine - Video-Only Asset Handling", () => {
|
|
47
|
-
test("audioRendition returns undefined for video-only asset", ({
|
|
48
|
-
videoOnlyAssetEngine,
|
|
49
|
-
expect,
|
|
50
|
-
}) => {
|
|
51
|
-
// This is the core fix - should return undefined, not {src: "..."}
|
|
52
|
-
const audioRendition = videoOnlyAssetEngine.audioRendition;
|
|
53
|
-
expect(audioRendition).toBeUndefined();
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
test("videoRendition returns valid object for video-only asset", ({
|
|
57
|
-
videoOnlyAssetEngine,
|
|
58
|
-
expect,
|
|
59
|
-
}) => {
|
|
60
|
-
const videoRendition = videoOnlyAssetEngine.videoRendition;
|
|
61
|
-
expect(videoRendition).toBeDefined();
|
|
62
|
-
expect(videoRendition?.trackId).toBe(1);
|
|
63
|
-
expect(videoRendition?.src).toBe("test-video-only.mp4");
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
test("getAudioRendition returns undefined for video-only asset", ({
|
|
67
|
-
videoOnlyAssetEngine,
|
|
68
|
-
expect,
|
|
69
|
-
}) => {
|
|
70
|
-
// New API behavior - should return undefined gracefully
|
|
71
|
-
const result = videoOnlyAssetEngine.getAudioRendition();
|
|
72
|
-
expect(result).toBeUndefined();
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
test("original error scenario is prevented", ({
|
|
76
|
-
videoOnlyAssetEngine,
|
|
77
|
-
expect,
|
|
78
|
-
}) => {
|
|
79
|
-
// This is the exact scenario that caused the original error:
|
|
80
|
-
// "computeSegmentId: trackId not found for rendition {\"src\":\"uuid\"}"
|
|
81
|
-
|
|
82
|
-
const audioRendition = videoOnlyAssetEngine.getAudioRendition();
|
|
83
|
-
|
|
84
|
-
// Before fix: audioRendition would be {trackId: undefined, src: "..."}
|
|
85
|
-
// After fix: audioRendition should be undefined
|
|
86
|
-
expect(audioRendition).toBeUndefined();
|
|
87
|
-
|
|
88
|
-
// This prevents the downstream error where trackId was missing entirely
|
|
89
|
-
if (audioRendition !== undefined) {
|
|
90
|
-
// If audioRendition exists, it should have a valid trackId
|
|
91
|
-
expect(audioRendition.trackId).toBeDefined();
|
|
92
|
-
expect(typeof audioRendition.trackId).toBe("number");
|
|
93
|
-
}
|
|
94
|
-
});
|
|
95
|
-
});
|
|
@@ -1,184 +0,0 @@
|
|
|
1
|
-
import { Task } from "@lit/task";
|
|
2
|
-
|
|
3
|
-
import { EF_INTERACTIVE } from "../../../EF_INTERACTIVE.js";
|
|
4
|
-
import { LRUCache } from "../../../utils/LRUCache.js";
|
|
5
|
-
import { type EFMedia, IgnorableError } from "../../EFMedia.js";
|
|
6
|
-
|
|
7
|
-
// DECAY_WEIGHT constant - same as original
|
|
8
|
-
const DECAY_WEIGHT = 0.8;
|
|
9
|
-
|
|
10
|
-
export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
|
|
11
|
-
// Internal cache for this task instance (same as original #byteTimeDomainCache)
|
|
12
|
-
const cache = new LRUCache<string, Uint8Array>(1000);
|
|
13
|
-
|
|
14
|
-
return new Task(element, {
|
|
15
|
-
autoRun: EF_INTERACTIVE,
|
|
16
|
-
onError: (error) => {
|
|
17
|
-
if (error instanceof IgnorableError) {
|
|
18
|
-
console.info("byteTimeDomainTask skipped: no audio track");
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
console.error("byteTimeDomainTask error", error);
|
|
22
|
-
},
|
|
23
|
-
args: () =>
|
|
24
|
-
[
|
|
25
|
-
element.currentSourceTimeMs,
|
|
26
|
-
element.fftSize,
|
|
27
|
-
element.fftDecay,
|
|
28
|
-
element.fftGain,
|
|
29
|
-
element.shouldInterpolateFrequencies,
|
|
30
|
-
] as const,
|
|
31
|
-
task: async (_, { signal }) => {
|
|
32
|
-
if (element.currentSourceTimeMs < 0) return null;
|
|
33
|
-
|
|
34
|
-
const currentTimeMs = element.currentSourceTimeMs;
|
|
35
|
-
|
|
36
|
-
// Calculate exact audio window needed based on fftDecay and frame timing
|
|
37
|
-
const frameIntervalMs = 1000 / 30; // 33.33ms per frame
|
|
38
|
-
|
|
39
|
-
// Need audio from earliest frame to current frame
|
|
40
|
-
const earliestFrameMs =
|
|
41
|
-
currentTimeMs - (element.fftDecay - 1) * frameIntervalMs;
|
|
42
|
-
const fromMs = Math.max(0, earliestFrameMs);
|
|
43
|
-
const maxToMs = currentTimeMs + frameIntervalMs; // Include current frame
|
|
44
|
-
const videoDurationMs = element.intrinsicDurationMs || 0;
|
|
45
|
-
const toMs =
|
|
46
|
-
videoDurationMs > 0 ? Math.min(maxToMs, videoDurationMs) : maxToMs;
|
|
47
|
-
|
|
48
|
-
// If the clamping results in an invalid range (seeking beyond the end), skip analysis silently
|
|
49
|
-
if (fromMs >= toMs) {
|
|
50
|
-
return null;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Check cache early - before expensive audio fetching
|
|
54
|
-
// Use a preliminary cache key that doesn't depend on actual startOffsetMs from audio span
|
|
55
|
-
const preliminaryCacheKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftDecay}:${element.fftGain}:${fromMs}:${currentTimeMs}`;
|
|
56
|
-
const cachedData = cache.get(preliminaryCacheKey);
|
|
57
|
-
if (cachedData) {
|
|
58
|
-
return cachedData;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
const { fetchAudioSpanningTime: fetchAudioSpan } = await import(
|
|
62
|
-
"../shared/AudioSpanUtils.ts"
|
|
63
|
-
);
|
|
64
|
-
const audioSpan = await fetchAudioSpan(element, fromMs, toMs, signal);
|
|
65
|
-
|
|
66
|
-
if (!audioSpan || !audioSpan.blob) {
|
|
67
|
-
console.warn("Time domain analysis skipped: no audio data available");
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// Decode the real audio data
|
|
72
|
-
const tempAudioContext = new OfflineAudioContext(2, 48000, 48000);
|
|
73
|
-
const arrayBuffer = await audioSpan.blob.arrayBuffer();
|
|
74
|
-
const audioBuffer = await tempAudioContext.decodeAudioData(arrayBuffer);
|
|
75
|
-
|
|
76
|
-
// Use actual startOffset from audioSpan (relative to requested time)
|
|
77
|
-
const startOffsetMs = audioSpan.startMs;
|
|
78
|
-
|
|
79
|
-
// Process multiple frames with decay, similar to the reference code
|
|
80
|
-
const framesData = await Promise.all(
|
|
81
|
-
Array.from({ length: element.fftDecay }, async (_, frameIndex) => {
|
|
82
|
-
const frameOffset = frameIndex * (1000 / 30);
|
|
83
|
-
const startTime = Math.max(
|
|
84
|
-
0,
|
|
85
|
-
(currentTimeMs - frameOffset - startOffsetMs) / 1000,
|
|
86
|
-
);
|
|
87
|
-
|
|
88
|
-
const cacheKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftGain}:${startOffsetMs}:${startTime}`;
|
|
89
|
-
const cachedFrame = cache.get(cacheKey);
|
|
90
|
-
if (cachedFrame) {
|
|
91
|
-
return cachedFrame;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
let audioContext: OfflineAudioContext;
|
|
95
|
-
try {
|
|
96
|
-
audioContext = new OfflineAudioContext(2, 48000 * (1 / 30), 48000);
|
|
97
|
-
} catch (error) {
|
|
98
|
-
throw new Error(
|
|
99
|
-
`[EFMedia.byteTimeDomainTask] Failed to create OfflineAudioContext(2, ${48000 * (1 / 30)}, 48000) for frame ${frameIndex} at time ${startTime}s: ${error instanceof Error ? error.message : String(error)}. This is for audio time domain analysis.`,
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const source = audioContext.createBufferSource();
|
|
104
|
-
source.buffer = audioBuffer;
|
|
105
|
-
|
|
106
|
-
// Create analyzer for PCM data
|
|
107
|
-
const analyser = audioContext.createAnalyser();
|
|
108
|
-
analyser.fftSize = element.fftSize; // Ensure enough samples
|
|
109
|
-
analyser.minDecibels = -90;
|
|
110
|
-
analyser.maxDecibels = -20;
|
|
111
|
-
|
|
112
|
-
const gainNode = audioContext.createGain();
|
|
113
|
-
gainNode.gain.value = element.fftGain; // Amplify the signal
|
|
114
|
-
|
|
115
|
-
source.connect(gainNode);
|
|
116
|
-
gainNode.connect(analyser);
|
|
117
|
-
analyser.connect(audioContext.destination);
|
|
118
|
-
|
|
119
|
-
source.start(0, startTime, 1 / 30);
|
|
120
|
-
|
|
121
|
-
const dataLength = analyser.fftSize / 2;
|
|
122
|
-
try {
|
|
123
|
-
await audioContext.startRendering();
|
|
124
|
-
const frameData = new Uint8Array(dataLength);
|
|
125
|
-
analyser.getByteTimeDomainData(frameData);
|
|
126
|
-
|
|
127
|
-
// const points = frameData;
|
|
128
|
-
// Calculate RMS and midpoint values
|
|
129
|
-
const points = new Uint8Array(dataLength);
|
|
130
|
-
for (let i = 0; i < dataLength; i++) {
|
|
131
|
-
const pointSamples = frameData.slice(
|
|
132
|
-
i * (frameData.length / dataLength),
|
|
133
|
-
(i + 1) * (frameData.length / dataLength),
|
|
134
|
-
);
|
|
135
|
-
|
|
136
|
-
// Calculate RMS while preserving sign
|
|
137
|
-
const rms = Math.sqrt(
|
|
138
|
-
pointSamples.reduce((sum, sample) => {
|
|
139
|
-
const normalized = (sample - 128) / 128;
|
|
140
|
-
return sum + normalized * normalized;
|
|
141
|
-
}, 0) / pointSamples.length,
|
|
142
|
-
);
|
|
143
|
-
|
|
144
|
-
// Get average sign of the samples to determine direction
|
|
145
|
-
const avgSign = Math.sign(
|
|
146
|
-
pointSamples.reduce((sum, sample) => sum + (sample - 128), 0),
|
|
147
|
-
);
|
|
148
|
-
|
|
149
|
-
// Convert RMS back to byte range, preserving direction
|
|
150
|
-
points[i] = Math.min(255, Math.round(128 + avgSign * rms * 128));
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
cache.set(cacheKey, points);
|
|
154
|
-
return points;
|
|
155
|
-
} finally {
|
|
156
|
-
source.disconnect();
|
|
157
|
-
analyser.disconnect();
|
|
158
|
-
}
|
|
159
|
-
}),
|
|
160
|
-
);
|
|
161
|
-
|
|
162
|
-
// Combine frames with decay weighting
|
|
163
|
-
const frameLength = framesData[0]?.length ?? 0;
|
|
164
|
-
const smoothedData = new Uint8Array(frameLength);
|
|
165
|
-
|
|
166
|
-
for (let i = 0; i < frameLength; i++) {
|
|
167
|
-
let weightedSum = 0;
|
|
168
|
-
let weightSum = 0;
|
|
169
|
-
|
|
170
|
-
framesData.forEach((frame: Uint8Array, frameIndex: number) => {
|
|
171
|
-
const decayWeight = DECAY_WEIGHT ** frameIndex;
|
|
172
|
-
weightedSum += (frame[i] ?? 0) * decayWeight;
|
|
173
|
-
weightSum += decayWeight;
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
// Cache with the preliminary key so future requests can skip audio fetching
|
|
180
|
-
cache.set(preliminaryCacheKey, smoothedData);
|
|
181
|
-
return smoothedData;
|
|
182
|
-
},
|
|
183
|
-
});
|
|
184
|
-
}
|