@editframe/elements 0.19.4-beta.0 → 0.20.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/elements/ContextProxiesController.d.ts +40 -0
- package/dist/elements/ContextProxiesController.js +69 -0
- package/dist/elements/EFCaptions.d.ts +45 -6
- package/dist/elements/EFCaptions.js +220 -26
- package/dist/elements/EFImage.js +4 -1
- package/dist/elements/EFMedia/AssetIdMediaEngine.d.ts +2 -1
- package/dist/elements/EFMedia/AssetIdMediaEngine.js +9 -0
- package/dist/elements/EFMedia/AssetMediaEngine.d.ts +1 -0
- package/dist/elements/EFMedia/AssetMediaEngine.js +11 -0
- package/dist/elements/EFMedia/BaseMediaEngine.d.ts +13 -1
- package/dist/elements/EFMedia/BaseMediaEngine.js +9 -0
- package/dist/elements/EFMedia/JitMediaEngine.d.ts +7 -1
- package/dist/elements/EFMedia/JitMediaEngine.js +15 -0
- package/dist/elements/EFMedia/audioTasks/makeAudioBufferTask.js +2 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.js +2 -0
- package/dist/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.d.ts +1 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.js +3 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioInputTask.js +1 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.d.ts +1 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.js +6 -5
- package/dist/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.js +3 -1
- package/dist/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.js +2 -0
- package/dist/elements/EFMedia/shared/AudioSpanUtils.js +2 -2
- package/dist/elements/EFMedia/shared/GlobalInputCache.d.ts +39 -0
- package/dist/elements/EFMedia/shared/GlobalInputCache.js +57 -0
- package/dist/elements/EFMedia/shared/ThumbnailExtractor.d.ts +27 -0
- package/dist/elements/EFMedia/shared/ThumbnailExtractor.js +106 -0
- package/dist/elements/EFMedia/tasks/makeMediaEngineTask.js +1 -1
- package/dist/elements/EFMedia.d.ts +2 -2
- package/dist/elements/EFMedia.js +25 -1
- package/dist/elements/EFSurface.browsertest.d.ts +0 -0
- package/dist/elements/EFSurface.d.ts +30 -0
- package/dist/elements/EFSurface.js +96 -0
- package/dist/elements/EFTemporal.js +7 -6
- package/dist/elements/EFThumbnailStrip.browsertest.d.ts +0 -0
- package/dist/elements/EFThumbnailStrip.d.ts +86 -0
- package/dist/elements/EFThumbnailStrip.js +490 -0
- package/dist/elements/EFThumbnailStrip.media-engine.browsertest.d.ts +0 -0
- package/dist/elements/EFTimegroup.d.ts +6 -1
- package/dist/elements/EFTimegroup.js +53 -11
- package/dist/elements/updateAnimations.browsertest.d.ts +13 -0
- package/dist/elements/updateAnimations.d.ts +5 -0
- package/dist/elements/updateAnimations.js +37 -13
- package/dist/getRenderInfo.js +1 -1
- package/dist/gui/ContextMixin.js +27 -14
- package/dist/gui/EFControls.browsertest.d.ts +0 -0
- package/dist/gui/EFControls.d.ts +38 -0
- package/dist/gui/EFControls.js +51 -0
- package/dist/gui/EFFilmstrip.d.ts +40 -1
- package/dist/gui/EFFilmstrip.js +240 -3
- package/dist/gui/EFPreview.js +2 -1
- package/dist/gui/EFScrubber.d.ts +6 -5
- package/dist/gui/EFScrubber.js +31 -21
- package/dist/gui/EFTimeDisplay.browsertest.d.ts +0 -0
- package/dist/gui/EFTimeDisplay.d.ts +2 -6
- package/dist/gui/EFTimeDisplay.js +13 -23
- package/dist/gui/TWMixin.js +1 -1
- package/dist/gui/currentTimeContext.d.ts +3 -0
- package/dist/gui/currentTimeContext.js +3 -0
- package/dist/gui/durationContext.d.ts +3 -0
- package/dist/gui/durationContext.js +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +4 -1
- package/dist/style.css +1 -1
- package/dist/transcoding/types/index.d.ts +11 -0
- package/dist/utils/LRUCache.d.ts +46 -0
- package/dist/utils/LRUCache.js +382 -1
- package/dist/utils/LRUCache.test.d.ts +1 -0
- package/package.json +2 -2
- package/src/elements/ContextProxiesController.ts +124 -0
- package/src/elements/EFCaptions.browsertest.ts +1820 -0
- package/src/elements/EFCaptions.ts +373 -36
- package/src/elements/EFImage.ts +4 -1
- package/src/elements/EFMedia/AssetIdMediaEngine.ts +30 -1
- package/src/elements/EFMedia/AssetMediaEngine.ts +33 -0
- package/src/elements/EFMedia/BaseMediaEngine.browsertest.ts +3 -8
- package/src/elements/EFMedia/BaseMediaEngine.ts +35 -0
- package/src/elements/EFMedia/JitMediaEngine.ts +34 -0
- package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.ts +6 -5
- package/src/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts +5 -0
- package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts +8 -5
- package/src/elements/EFMedia/audioTasks/makeAudioInputTask.ts +5 -5
- package/src/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.ts +11 -12
- package/src/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.ts +7 -4
- package/src/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts +5 -0
- package/src/elements/EFMedia/shared/AudioSpanUtils.ts +2 -2
- package/src/elements/EFMedia/shared/GlobalInputCache.ts +77 -0
- package/src/elements/EFMedia/shared/RenditionHelpers.browsertest.ts +2 -2
- package/src/elements/EFMedia/shared/RenditionHelpers.ts +2 -2
- package/src/elements/EFMedia/shared/ThumbnailExtractor.ts +227 -0
- package/src/elements/EFMedia/tasks/makeMediaEngineTask.ts +1 -1
- package/src/elements/EFMedia.ts +38 -1
- package/src/elements/EFSurface.browsertest.ts +155 -0
- package/src/elements/EFSurface.ts +141 -0
- package/src/elements/EFTemporal.ts +14 -8
- package/src/elements/EFThumbnailStrip.browsertest.ts +591 -0
- package/src/elements/EFThumbnailStrip.media-engine.browsertest.ts +713 -0
- package/src/elements/EFThumbnailStrip.ts +905 -0
- package/src/elements/EFTimegroup.browsertest.ts +56 -7
- package/src/elements/EFTimegroup.ts +88 -16
- package/src/elements/updateAnimations.browsertest.ts +333 -11
- package/src/elements/updateAnimations.ts +68 -19
- package/src/gui/ContextMixin.browsertest.ts +0 -25
- package/src/gui/ContextMixin.ts +44 -20
- package/src/gui/EFControls.browsertest.ts +175 -0
- package/src/gui/EFControls.ts +84 -0
- package/src/gui/EFFilmstrip.ts +323 -4
- package/src/gui/EFPreview.ts +2 -1
- package/src/gui/EFScrubber.ts +29 -25
- package/src/gui/EFTimeDisplay.browsertest.ts +237 -0
- package/src/gui/EFTimeDisplay.ts +12 -40
- package/src/gui/currentTimeContext.ts +5 -0
- package/src/gui/durationContext.ts +3 -0
- package/src/transcoding/types/index.ts +13 -0
- package/src/utils/LRUCache.test.ts +272 -0
- package/src/utils/LRUCache.ts +543 -0
- package/test/__cache__/GET__api_v1_transcode_high_1_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__26197f6f7c46cacb0a71134131c3f775/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_high_1_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__26197f6f7c46cacb0a71134131c3f775/metadata.json +1 -1
- package/test/__cache__/GET__api_v1_transcode_high_2_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__4cb6774cd3650ccf59c8f8dc6678c0b9/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_high_2_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__4cb6774cd3650ccf59c8f8dc6678c0b9/metadata.json +1 -1
- package/test/__cache__/GET__api_v1_transcode_high_3_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0b3b2b1c8933f7fcf8a9ecaa88d58b41/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_high_3_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0b3b2b1c8933f7fcf8a9ecaa88d58b41/metadata.json +1 -1
- package/test/__cache__/GET__api_v1_transcode_high_4_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a6fb05a22b18d850f7f2950bbcdbdeed/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_high_4_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a6fb05a22b18d850f7f2950bbcdbdeed/metadata.json +1 -1
- package/test/__cache__/GET__api_v1_transcode_high_5_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a50058c7c3602e90879fe3428ed891f4/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_high_5_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a50058c7c3602e90879fe3428ed891f4/metadata.json +1 -1
- package/test/__cache__/GET__api_v1_transcode_high_init_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0798c479b44aaeef850609a430f6e613/data.bin +0 -0
- package/test/__cache__/GET__api_v1_transcode_manifest_json_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__3be92a0437de726b431ed5af2369158a/data.bin +1 -1
- package/test/__cache__/GET__api_v1_transcode_manifest_json_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__3be92a0437de726b431ed5af2369158a/metadata.json +1 -1
- package/types.json +1 -1
- package/dist/transcoding/cache/CacheManager.d.ts +0 -73
- package/src/transcoding/cache/CacheManager.ts +0 -208
|
@@ -7,6 +7,7 @@ import { BaseMediaEngine, mediaCache } from "./BaseMediaEngine.js";
|
|
|
7
7
|
const test = baseTest.extend<{}>({});
|
|
8
8
|
|
|
9
9
|
// Test implementation of BaseMediaEngine for testing
|
|
10
|
+
// @ts-expect-error missing implementations
|
|
10
11
|
class TestMediaEngine extends BaseMediaEngine {
|
|
11
12
|
fetchMediaSegment = vi.fn();
|
|
12
13
|
public host: EFMedia;
|
|
@@ -221,14 +222,8 @@ describe("BaseMediaEngine abort signal handling", () => {
|
|
|
221
222
|
const host = {
|
|
222
223
|
fetch: vi
|
|
223
224
|
.fn()
|
|
224
|
-
.mockImplementation(
|
|
225
|
-
() =>
|
|
226
|
-
new Promise((resolve) =>
|
|
227
|
-
setTimeout(
|
|
228
|
-
() => resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
|
|
229
|
-
100,
|
|
230
|
-
),
|
|
231
|
-
),
|
|
225
|
+
.mockImplementation(() =>
|
|
226
|
+
Promise.resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
|
|
232
227
|
),
|
|
233
228
|
} as any;
|
|
234
229
|
|
|
@@ -2,10 +2,12 @@ import { RequestDeduplicator } from "../../transcoding/cache/RequestDeduplicator
|
|
|
2
2
|
import type {
|
|
3
3
|
AudioRendition,
|
|
4
4
|
SegmentTimeRange,
|
|
5
|
+
ThumbnailResult,
|
|
5
6
|
VideoRendition,
|
|
6
7
|
} from "../../transcoding/types";
|
|
7
8
|
import { SizeAwareLRUCache } from "../../utils/LRUCache.js";
|
|
8
9
|
import type { EFMedia } from "../EFMedia.js";
|
|
10
|
+
import type { MediaRendition } from "./shared/MediaTaskUtils.js";
|
|
9
11
|
|
|
10
12
|
// Global instances shared across all media engines
|
|
11
13
|
export const mediaCache = new SizeAwareLRUCache<string>(100 * 1024 * 1024); // 100MB cache limit
|
|
@@ -205,6 +207,16 @@ export abstract class BaseMediaEngine {
|
|
|
205
207
|
rendition: { trackId: number | undefined; src: string },
|
|
206
208
|
): Promise<ArrayBuffer>;
|
|
207
209
|
|
|
210
|
+
abstract fetchInitSegment(
|
|
211
|
+
rendition: { trackId: number | undefined; src: string },
|
|
212
|
+
signal: AbortSignal,
|
|
213
|
+
): Promise<ArrayBuffer>;
|
|
214
|
+
|
|
215
|
+
abstract computeSegmentId(
|
|
216
|
+
desiredSeekTimeMs: number,
|
|
217
|
+
rendition: MediaRendition,
|
|
218
|
+
): number | undefined;
|
|
219
|
+
|
|
208
220
|
/**
|
|
209
221
|
* Fetch media segment with built-in deduplication
|
|
210
222
|
* Now uses global deduplication for all requests
|
|
@@ -387,4 +399,27 @@ export abstract class BaseMediaEngine {
|
|
|
387
399
|
segmentIds.filter((id) => this.isSegmentCached(id, rendition)),
|
|
388
400
|
);
|
|
389
401
|
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Extract thumbnail canvases at multiple timestamps efficiently
|
|
405
|
+
* Default implementation provides helpful error information
|
|
406
|
+
*/
|
|
407
|
+
async extractThumbnails(
|
|
408
|
+
timestamps: number[],
|
|
409
|
+
): Promise<(ThumbnailResult | null)[]> {
|
|
410
|
+
const engineName = this.constructor.name;
|
|
411
|
+
console.warn(
|
|
412
|
+
`${engineName}: extractThumbnails not properly implemented. ` +
|
|
413
|
+
"This MediaEngine type does not support thumbnail generation. " +
|
|
414
|
+
"Supported engines: JitMediaEngine. " +
|
|
415
|
+
`Requested ${timestamps.length} thumbnail${timestamps.length === 1 ? "" : "s"}.`,
|
|
416
|
+
);
|
|
417
|
+
return timestamps.map(() => null);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
abstract convertToSegmentRelativeTimestamps(
|
|
421
|
+
globalTimestamps: number[],
|
|
422
|
+
segmentId: number,
|
|
423
|
+
rendition: VideoRendition,
|
|
424
|
+
): number[];
|
|
390
425
|
}
|
|
@@ -2,16 +2,19 @@ import type {
|
|
|
2
2
|
AudioRendition,
|
|
3
3
|
MediaEngine,
|
|
4
4
|
RenditionId,
|
|
5
|
+
ThumbnailResult,
|
|
5
6
|
VideoRendition,
|
|
6
7
|
} from "../../transcoding/types";
|
|
7
8
|
import type { ManifestResponse } from "../../transcoding/types/index.js";
|
|
8
9
|
import type { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
|
|
9
10
|
import type { EFMedia } from "../EFMedia.js";
|
|
10
11
|
import { BaseMediaEngine } from "./BaseMediaEngine";
|
|
12
|
+
import { ThumbnailExtractor } from "./shared/ThumbnailExtractor.js";
|
|
11
13
|
|
|
12
14
|
export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
|
|
13
15
|
private urlGenerator: UrlGenerator;
|
|
14
16
|
private data: ManifestResponse = {} as ManifestResponse;
|
|
17
|
+
private thumbnailExtractor: ThumbnailExtractor;
|
|
15
18
|
|
|
16
19
|
static async fetch(host: EFMedia, urlGenerator: UrlGenerator, url: string) {
|
|
17
20
|
const engine = new JitMediaEngine(host, urlGenerator);
|
|
@@ -23,6 +26,7 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
|
|
|
23
26
|
constructor(host: EFMedia, urlGenerator: UrlGenerator) {
|
|
24
27
|
super(host);
|
|
25
28
|
this.urlGenerator = urlGenerator;
|
|
29
|
+
this.thumbnailExtractor = new ThumbnailExtractor(this);
|
|
26
30
|
}
|
|
27
31
|
|
|
28
32
|
get durationMs() {
|
|
@@ -204,4 +208,34 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
|
|
|
204
208
|
maxAudioBufferFetches: 3,
|
|
205
209
|
};
|
|
206
210
|
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Extract thumbnail canvases using same rendition priority as video playback for frame alignment
|
|
214
|
+
*/
|
|
215
|
+
async extractThumbnails(
|
|
216
|
+
timestamps: number[],
|
|
217
|
+
): Promise<(ThumbnailResult | null)[]> {
|
|
218
|
+
const mainRendition = this.videoRendition;
|
|
219
|
+
const scrubRendition = this.getScrubVideoRendition();
|
|
220
|
+
|
|
221
|
+
const rendition = mainRendition || scrubRendition;
|
|
222
|
+
|
|
223
|
+
if (!rendition) {
|
|
224
|
+
return timestamps.map(() => null);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return this.thumbnailExtractor.extractThumbnails(
|
|
228
|
+
timestamps,
|
|
229
|
+
rendition,
|
|
230
|
+
this.durationMs,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
convertToSegmentRelativeTimestamps(
|
|
235
|
+
globalTimestamps: number[],
|
|
236
|
+
_segmentId: number,
|
|
237
|
+
_rendition: VideoRendition,
|
|
238
|
+
): number[] {
|
|
239
|
+
return globalTimestamps.map((timestamp) => timestamp / 1000);
|
|
240
|
+
}
|
|
207
241
|
}
|
|
@@ -42,13 +42,15 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
|
|
|
42
42
|
task: async ([seekTimeMs], { signal }) => {
|
|
43
43
|
// Skip buffering entirely in rendering mode
|
|
44
44
|
if (EF_RENDERING()) {
|
|
45
|
-
return currentState;
|
|
45
|
+
return currentState;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
// Get media engine to potentially override buffer configuration
|
|
49
48
|
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
50
49
|
|
|
51
|
-
|
|
50
|
+
if (!mediaEngine.audioRendition) {
|
|
51
|
+
return currentState;
|
|
52
|
+
}
|
|
53
|
+
|
|
52
54
|
const engineConfig = mediaEngine.getBufferConfig();
|
|
53
55
|
const bufferDurationMs = engineConfig.audioBufferDurationMs;
|
|
54
56
|
const maxParallelFetches = engineConfig.maxAudioBufferFetches;
|
|
@@ -85,11 +87,10 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
|
|
|
85
87
|
return mediaEngine.isSegmentCached(segmentId, rendition);
|
|
86
88
|
},
|
|
87
89
|
getRendition: async () => {
|
|
88
|
-
// Get real audio rendition from media engine
|
|
89
90
|
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
90
91
|
const audioRendition = mediaEngine.audioRendition;
|
|
91
92
|
if (!audioRendition) {
|
|
92
|
-
throw new Error("
|
|
93
|
+
throw new Error("No audio track available in source");
|
|
93
94
|
}
|
|
94
95
|
return audioRendition;
|
|
95
96
|
},
|
|
@@ -95,6 +95,11 @@ export function makeAudioFrequencyAnalysisTask(element: EFMedia) {
|
|
|
95
95
|
task: async (_, { signal }) => {
|
|
96
96
|
if (element.currentSourceTimeMs < 0) return null;
|
|
97
97
|
|
|
98
|
+
const mediaEngine = element.mediaEngineTask.value;
|
|
99
|
+
if (!mediaEngine?.audioRendition) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
98
103
|
const currentTimeMs = element.currentSourceTimeMs;
|
|
99
104
|
|
|
100
105
|
// Calculate exact audio window needed based on fftDecay and frame timing
|
|
@@ -5,7 +5,7 @@ import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
|
|
|
5
5
|
|
|
6
6
|
export const makeAudioInitSegmentFetchTask = (
|
|
7
7
|
host: EFMedia,
|
|
8
|
-
): Task<readonly [MediaEngine | undefined], ArrayBuffer> => {
|
|
8
|
+
): Task<readonly [MediaEngine | undefined], ArrayBuffer | undefined> => {
|
|
9
9
|
return new Task(host, {
|
|
10
10
|
args: () => [host.mediaEngineTask.value] as const,
|
|
11
11
|
onError: (error) => {
|
|
@@ -14,10 +14,13 @@ export const makeAudioInitSegmentFetchTask = (
|
|
|
14
14
|
onComplete: (_value) => {},
|
|
15
15
|
task: async ([_mediaEngine], { signal }) => {
|
|
16
16
|
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
|
|
18
|
+
const audioRendition = mediaEngine.audioRendition;
|
|
19
|
+
if (!audioRendition) {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
return mediaEngine.fetchInitSegment(audioRendition, signal);
|
|
21
24
|
},
|
|
22
25
|
});
|
|
23
26
|
};
|
|
@@ -19,20 +19,20 @@ export const makeAudioInputTask = (host: EFMedia): InputTask => {
|
|
|
19
19
|
onComplete: (_value) => {},
|
|
20
20
|
task: async (_, { signal }) => {
|
|
21
21
|
const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
|
|
22
|
-
signal.throwIfAborted();
|
|
22
|
+
signal.throwIfAborted();
|
|
23
23
|
const segment = await host.audioSegmentFetchTask.taskComplete;
|
|
24
|
-
signal.throwIfAborted();
|
|
24
|
+
signal.throwIfAborted();
|
|
25
|
+
|
|
25
26
|
if (!initSegment || !segment) {
|
|
26
|
-
throw new Error("
|
|
27
|
+
throw new Error("No audio track available in source");
|
|
27
28
|
}
|
|
28
29
|
|
|
29
|
-
// Get startTimeOffsetMs from the audio rendition if available
|
|
30
30
|
const mediaEngine = await host.mediaEngineTask.taskComplete;
|
|
31
31
|
const audioRendition = mediaEngine?.audioRendition;
|
|
32
32
|
const startTimeOffsetMs = audioRendition?.startTimeOffsetMs;
|
|
33
33
|
|
|
34
34
|
const arrayBuffer = await new Blob([initSegment, segment]).arrayBuffer();
|
|
35
|
-
signal.throwIfAborted();
|
|
35
|
+
signal.throwIfAborted();
|
|
36
36
|
return new BufferedSeekingInput(arrayBuffer, {
|
|
37
37
|
videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
|
|
38
38
|
audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
|
|
@@ -7,7 +7,7 @@ export const makeAudioSegmentFetchTask = (
|
|
|
7
7
|
host: EFMedia,
|
|
8
8
|
): Task<
|
|
9
9
|
readonly [MediaEngine | undefined, number | undefined],
|
|
10
|
-
ArrayBuffer
|
|
10
|
+
ArrayBuffer | undefined
|
|
11
11
|
> => {
|
|
12
12
|
return new Task(host, {
|
|
13
13
|
args: () =>
|
|
@@ -18,14 +18,18 @@ export const makeAudioSegmentFetchTask = (
|
|
|
18
18
|
onComplete: (_value) => {},
|
|
19
19
|
task: async (_, { signal }) => {
|
|
20
20
|
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
21
|
+
|
|
22
|
+
const audioRendition = mediaEngine.audioRendition;
|
|
23
|
+
if (!audioRendition) {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
21
27
|
const segmentId = await host.audioSegmentIdTask.taskComplete;
|
|
22
28
|
if (segmentId === undefined) {
|
|
23
|
-
// Provide more context in the error to help with debugging
|
|
24
|
-
const rendition = mediaEngine.audioRendition;
|
|
25
29
|
const debugInfo = {
|
|
26
|
-
hasRendition:
|
|
27
|
-
segmentDurationMs:
|
|
28
|
-
segmentDurationsMs:
|
|
30
|
+
hasRendition: true,
|
|
31
|
+
segmentDurationMs: audioRendition.segmentDurationMs,
|
|
32
|
+
segmentDurationsMs: audioRendition.segmentDurationsMs?.length || 0,
|
|
29
33
|
desiredSeekTimeMs: host.desiredSeekTimeMs,
|
|
30
34
|
intrinsicDurationMs: host.intrinsicDurationMs,
|
|
31
35
|
};
|
|
@@ -34,12 +38,7 @@ export const makeAudioSegmentFetchTask = (
|
|
|
34
38
|
);
|
|
35
39
|
}
|
|
36
40
|
|
|
37
|
-
|
|
38
|
-
return mediaEngine.fetchMediaSegment(
|
|
39
|
-
segmentId,
|
|
40
|
-
mediaEngine.getAudioRendition(),
|
|
41
|
-
signal,
|
|
42
|
-
);
|
|
41
|
+
return mediaEngine.fetchMediaSegment(segmentId, audioRendition, signal);
|
|
43
42
|
},
|
|
44
43
|
});
|
|
45
44
|
};
|
|
@@ -15,10 +15,13 @@ export const makeAudioSegmentIdTask = (
|
|
|
15
15
|
task: async ([, targetSeekTimeMs], { signal }) => {
|
|
16
16
|
const mediaEngine = await getLatestMediaEngine(host, signal);
|
|
17
17
|
signal.throwIfAborted(); // Abort if a new seek started
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
|
|
19
|
+
const audioRendition = mediaEngine.audioRendition;
|
|
20
|
+
if (!audioRendition) {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
return mediaEngine.computeSegmentId(targetSeekTimeMs, audioRendition);
|
|
22
25
|
},
|
|
23
26
|
});
|
|
24
27
|
};
|
|
@@ -27,6 +27,11 @@ export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
|
|
|
27
27
|
task: async (_, { signal }) => {
|
|
28
28
|
if (element.currentSourceTimeMs < 0) return null;
|
|
29
29
|
|
|
30
|
+
const mediaEngine = element.mediaEngineTask.value;
|
|
31
|
+
if (!mediaEngine?.audioRendition) {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
|
|
30
35
|
const currentTimeMs = element.currentSourceTimeMs;
|
|
31
36
|
|
|
32
37
|
// Calculate exact audio window needed based on fftDecay and frame timing
|
|
@@ -16,7 +16,7 @@ const fetchAudioSegmentData = async (
|
|
|
16
16
|
): Promise<Map<number, ArrayBuffer>> => {
|
|
17
17
|
const audioRendition = mediaEngine.audioRendition;
|
|
18
18
|
if (!audioRendition) {
|
|
19
|
-
throw new Error("
|
|
19
|
+
throw new Error("No audio track available in source");
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
const segmentData = new Map<number, ArrayBuffer>();
|
|
@@ -73,7 +73,7 @@ export const fetchAudioSpanningTime = async (
|
|
|
73
73
|
const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
|
|
74
74
|
|
|
75
75
|
if (!mediaEngine?.audioRendition) {
|
|
76
|
-
throw new Error("
|
|
76
|
+
throw new Error("No audio track available in source");
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
if (!initSegment) {
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { Input } from "mediabunny";
|
|
2
|
+
import { LRUCache } from "../../../utils/LRUCache.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Global cache for MediaBunny Input instances
|
|
6
|
+
* Shared across all MediaEngine instances to prevent duplicate decoding
|
|
7
|
+
* of the same segment data
|
|
8
|
+
*/
|
|
9
|
+
class GlobalInputCache {
|
|
10
|
+
private cache = new LRUCache<string, Input>(50); // 50 Input instances max
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Generate standardized cache key for Input objects
|
|
14
|
+
* Format: "input:{src}:{segmentId}:{renditionId}"
|
|
15
|
+
*/
|
|
16
|
+
private generateKey(
|
|
17
|
+
src: string,
|
|
18
|
+
segmentId: number,
|
|
19
|
+
renditionId?: string,
|
|
20
|
+
): string {
|
|
21
|
+
return `input:${src}:${segmentId}:${renditionId || "default"}`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Get cached Input object
|
|
26
|
+
*/
|
|
27
|
+
get(src: string, segmentId: number, renditionId?: string): Input | undefined {
|
|
28
|
+
const key = this.generateKey(src, segmentId, renditionId);
|
|
29
|
+
return this.cache.get(key);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Cache Input object
|
|
34
|
+
*/
|
|
35
|
+
set(
|
|
36
|
+
src: string,
|
|
37
|
+
segmentId: number,
|
|
38
|
+
input: Input,
|
|
39
|
+
renditionId?: string,
|
|
40
|
+
): void {
|
|
41
|
+
const key = this.generateKey(src, segmentId, renditionId);
|
|
42
|
+
this.cache.set(key, input);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Check if Input is cached
|
|
47
|
+
*/
|
|
48
|
+
has(src: string, segmentId: number, renditionId?: string): boolean {
|
|
49
|
+
const key = this.generateKey(src, segmentId, renditionId);
|
|
50
|
+
return this.cache.has(key);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Clear all cached Input objects
|
|
55
|
+
*/
|
|
56
|
+
clear(): void {
|
|
57
|
+
this.cache.clear();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Get cache statistics for debugging
|
|
62
|
+
*/
|
|
63
|
+
getStats() {
|
|
64
|
+
return {
|
|
65
|
+
size: this.cache.size,
|
|
66
|
+
cachedKeys: Array.from((this.cache as any).cache.keys()),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Single global instance shared across all MediaEngine instances
|
|
72
|
+
export const globalInputCache = new GlobalInputCache();
|
|
73
|
+
|
|
74
|
+
// Export for debugging (works in both browser and server)
|
|
75
|
+
(
|
|
76
|
+
globalThis as typeof globalThis & { debugInputCache: typeof globalInputCache }
|
|
77
|
+
).debugInputCache = globalInputCache;
|
|
@@ -107,7 +107,7 @@ describe("RenditionHelpers", () => {
|
|
|
107
107
|
expect,
|
|
108
108
|
}) => {
|
|
109
109
|
expect(() => getAudioRendition(mockMediaEngineWithoutAudio)).toThrow(
|
|
110
|
-
"
|
|
110
|
+
"No audio track available in source",
|
|
111
111
|
);
|
|
112
112
|
});
|
|
113
113
|
});
|
|
@@ -128,7 +128,7 @@ describe("RenditionHelpers", () => {
|
|
|
128
128
|
expect,
|
|
129
129
|
}) => {
|
|
130
130
|
expect(() => getVideoRendition(mockMediaEngineWithoutVideo)).toThrow(
|
|
131
|
-
"
|
|
131
|
+
"No video track available in source",
|
|
132
132
|
);
|
|
133
133
|
});
|
|
134
134
|
});
|
|
@@ -10,7 +10,7 @@ import type {
|
|
|
10
10
|
export const getAudioRendition = (mediaEngine: MediaEngine): AudioRendition => {
|
|
11
11
|
const audioRendition = mediaEngine.audioRendition;
|
|
12
12
|
if (!audioRendition) {
|
|
13
|
-
throw new Error("
|
|
13
|
+
throw new Error("No audio track available in source");
|
|
14
14
|
}
|
|
15
15
|
return audioRendition;
|
|
16
16
|
};
|
|
@@ -21,7 +21,7 @@ export const getAudioRendition = (mediaEngine: MediaEngine): AudioRendition => {
|
|
|
21
21
|
export const getVideoRendition = (mediaEngine: MediaEngine): VideoRendition => {
|
|
22
22
|
const videoRendition = mediaEngine.videoRendition;
|
|
23
23
|
if (!videoRendition) {
|
|
24
|
-
throw new Error("
|
|
24
|
+
throw new Error("No video track available in source");
|
|
25
25
|
}
|
|
26
26
|
return videoRendition;
|
|
27
27
|
};
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { ALL_FORMATS, BlobSource, CanvasSink, Input } from "mediabunny";
|
|
2
|
+
import type {
|
|
3
|
+
ThumbnailResult,
|
|
4
|
+
VideoRendition,
|
|
5
|
+
} from "../../../transcoding/types/index.js";
|
|
6
|
+
import type { BaseMediaEngine } from "../BaseMediaEngine.js";
|
|
7
|
+
import { globalInputCache } from "./GlobalInputCache.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Shared thumbnail extraction logic for all MediaEngine implementations
|
|
11
|
+
* Eliminates code duplication and provides consistent behavior
|
|
12
|
+
*/
|
|
13
|
+
export class ThumbnailExtractor {
|
|
14
|
+
constructor(private mediaEngine: BaseMediaEngine) {}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Extract thumbnails at multiple timestamps efficiently using segment batching
|
|
18
|
+
*/
|
|
19
|
+
async extractThumbnails(
|
|
20
|
+
timestamps: number[],
|
|
21
|
+
rendition: VideoRendition,
|
|
22
|
+
durationMs: number,
|
|
23
|
+
): Promise<(ThumbnailResult | null)[]> {
|
|
24
|
+
if (timestamps.length === 0) {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Validate and filter timestamps within bounds
|
|
29
|
+
const validTimestamps = timestamps.filter(
|
|
30
|
+
(timeMs) => timeMs >= 0 && timeMs <= durationMs,
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
if (validTimestamps.length === 0) {
|
|
34
|
+
console.warn(
|
|
35
|
+
`ThumbnailExtractor: All timestamps out of bounds (0-${durationMs}ms)`,
|
|
36
|
+
);
|
|
37
|
+
return timestamps.map(() => null);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Group timestamps by segment for batch processing
|
|
41
|
+
const segmentGroups = this.groupTimestampsBySegment(
|
|
42
|
+
validTimestamps,
|
|
43
|
+
rendition,
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
// Extract batched by segment using CanvasSink
|
|
47
|
+
const results = new Map<number, ThumbnailResult | null>();
|
|
48
|
+
|
|
49
|
+
for (const [segmentId, segmentTimestamps] of segmentGroups) {
|
|
50
|
+
try {
|
|
51
|
+
const segmentResults = await this.extractSegmentThumbnails(
|
|
52
|
+
segmentId,
|
|
53
|
+
segmentTimestamps,
|
|
54
|
+
rendition,
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
for (const [timestamp, thumbnail] of segmentResults) {
|
|
58
|
+
results.set(timestamp, thumbnail);
|
|
59
|
+
}
|
|
60
|
+
} catch (error) {
|
|
61
|
+
console.warn(
|
|
62
|
+
`ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
|
|
63
|
+
error,
|
|
64
|
+
);
|
|
65
|
+
// Mark all timestamps in this segment as failed
|
|
66
|
+
for (const timestamp of segmentTimestamps) {
|
|
67
|
+
results.set(timestamp, null);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Return in original order, null for any that failed or were out of bounds
|
|
73
|
+
return timestamps.map((t) => {
|
|
74
|
+
// If timestamp was out of bounds, return null
|
|
75
|
+
if (t < 0 || t > durationMs) {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
return results.get(t) || null;
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Group timestamps by segment ID for efficient batch processing
|
|
84
|
+
*/
|
|
85
|
+
private groupTimestampsBySegment(
|
|
86
|
+
timestamps: number[],
|
|
87
|
+
rendition: VideoRendition,
|
|
88
|
+
): Map<number, number[]> {
|
|
89
|
+
const segmentGroups = new Map<number, number[]>();
|
|
90
|
+
|
|
91
|
+
for (const timeMs of timestamps) {
|
|
92
|
+
try {
|
|
93
|
+
const segmentId = this.mediaEngine.computeSegmentId(timeMs, rendition);
|
|
94
|
+
if (segmentId !== undefined) {
|
|
95
|
+
if (!segmentGroups.has(segmentId)) {
|
|
96
|
+
segmentGroups.set(segmentId, []);
|
|
97
|
+
}
|
|
98
|
+
const segmentGroup = segmentGroups.get(segmentId) ?? [];
|
|
99
|
+
if (!segmentGroup) {
|
|
100
|
+
segmentGroups.set(segmentId, []);
|
|
101
|
+
}
|
|
102
|
+
segmentGroup.push(timeMs);
|
|
103
|
+
}
|
|
104
|
+
} catch (error) {
|
|
105
|
+
console.warn(
|
|
106
|
+
`ThumbnailExtractor: Could not compute segment for timestamp ${timeMs}:`,
|
|
107
|
+
error,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return segmentGroups;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Extract thumbnails for a specific segment using CanvasSink
|
|
117
|
+
*/
|
|
118
|
+
private async extractSegmentThumbnails(
|
|
119
|
+
segmentId: number,
|
|
120
|
+
timestamps: number[],
|
|
121
|
+
rendition: VideoRendition,
|
|
122
|
+
): Promise<Map<number, ThumbnailResult | null>> {
|
|
123
|
+
const results = new Map<number, ThumbnailResult | null>();
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
// Get segment data through existing media engine methods (uses caches)
|
|
127
|
+
const abortController = new AbortController();
|
|
128
|
+
const [initSegment, mediaSegment] = await Promise.all([
|
|
129
|
+
this.mediaEngine.fetchInitSegment(rendition, abortController.signal),
|
|
130
|
+
this.mediaEngine.fetchMediaSegment(segmentId, rendition),
|
|
131
|
+
]);
|
|
132
|
+
|
|
133
|
+
// Create Input for this segment using global shared cache
|
|
134
|
+
const segmentBlob = new Blob([initSegment, mediaSegment]);
|
|
135
|
+
|
|
136
|
+
let input = globalInputCache.get(rendition.src, segmentId, rendition.id);
|
|
137
|
+
if (!input) {
|
|
138
|
+
input = new Input({
|
|
139
|
+
formats: ALL_FORMATS,
|
|
140
|
+
source: new BlobSource(segmentBlob),
|
|
141
|
+
});
|
|
142
|
+
globalInputCache.set(rendition.src, segmentId, input, rendition.id);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Set up CanvasSink for batched extraction
|
|
146
|
+
const videoTrack = await input.getPrimaryVideoTrack();
|
|
147
|
+
if (!videoTrack) {
|
|
148
|
+
// No video track - return nulls for all timestamps
|
|
149
|
+
for (const timestamp of timestamps) {
|
|
150
|
+
results.set(timestamp, null);
|
|
151
|
+
}
|
|
152
|
+
return results;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const sink = new CanvasSink(videoTrack);
|
|
156
|
+
|
|
157
|
+
// Convert global timestamps to segment-relative (in seconds for mediabunny)
|
|
158
|
+
const relativeTimestamps = this.convertToSegmentRelativeTimestamps(
|
|
159
|
+
timestamps,
|
|
160
|
+
segmentId,
|
|
161
|
+
rendition,
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
// Batch extract all thumbnails for this segment
|
|
165
|
+
const timestampResults = [];
|
|
166
|
+
for await (const result of sink.canvasesAtTimestamps(
|
|
167
|
+
relativeTimestamps,
|
|
168
|
+
)) {
|
|
169
|
+
timestampResults.push(result);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Map results back to original timestamps
|
|
173
|
+
for (let i = 0; i < timestamps.length; i++) {
|
|
174
|
+
const globalTimestamp = timestamps[i];
|
|
175
|
+
if (globalTimestamp === undefined) {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const result = timestampResults[i];
|
|
180
|
+
|
|
181
|
+
if (result?.canvas) {
|
|
182
|
+
const canvas = result.canvas;
|
|
183
|
+
if (
|
|
184
|
+
canvas instanceof HTMLCanvasElement ||
|
|
185
|
+
canvas instanceof OffscreenCanvas
|
|
186
|
+
) {
|
|
187
|
+
results.set(globalTimestamp, {
|
|
188
|
+
timestamp: globalTimestamp,
|
|
189
|
+
thumbnail: canvas,
|
|
190
|
+
});
|
|
191
|
+
} else {
|
|
192
|
+
results.set(globalTimestamp, null);
|
|
193
|
+
}
|
|
194
|
+
} else {
|
|
195
|
+
results.set(globalTimestamp, null);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
} catch (error) {
|
|
199
|
+
console.error(
|
|
200
|
+
`ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
|
|
201
|
+
error,
|
|
202
|
+
);
|
|
203
|
+
// Return nulls for all timestamps on error
|
|
204
|
+
for (const timestamp of timestamps) {
|
|
205
|
+
results.set(timestamp, null);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return results;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Convert global timestamps to segment-relative timestamps for mediabunny
|
|
214
|
+
* This is where the main difference between JIT and Asset engines lies
|
|
215
|
+
*/
|
|
216
|
+
private convertToSegmentRelativeTimestamps(
|
|
217
|
+
globalTimestamps: number[],
|
|
218
|
+
segmentId: number,
|
|
219
|
+
rendition: VideoRendition,
|
|
220
|
+
): number[] {
|
|
221
|
+
return this.mediaEngine.convertToSegmentRelativeTimestamps(
|
|
222
|
+
globalTimestamps,
|
|
223
|
+
segmentId,
|
|
224
|
+
rendition,
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
}
|