@editframe/elements 0.19.4-beta.0 → 0.20.1-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/elements/ContextProxiesController.d.ts +40 -0
  2. package/dist/elements/ContextProxiesController.js +69 -0
  3. package/dist/elements/EFCaptions.d.ts +45 -6
  4. package/dist/elements/EFCaptions.js +220 -26
  5. package/dist/elements/EFImage.js +4 -1
  6. package/dist/elements/EFMedia/AssetIdMediaEngine.d.ts +2 -1
  7. package/dist/elements/EFMedia/AssetIdMediaEngine.js +9 -0
  8. package/dist/elements/EFMedia/AssetMediaEngine.d.ts +1 -0
  9. package/dist/elements/EFMedia/AssetMediaEngine.js +11 -0
  10. package/dist/elements/EFMedia/BaseMediaEngine.d.ts +13 -1
  11. package/dist/elements/EFMedia/BaseMediaEngine.js +9 -0
  12. package/dist/elements/EFMedia/JitMediaEngine.d.ts +7 -1
  13. package/dist/elements/EFMedia/JitMediaEngine.js +15 -0
  14. package/dist/elements/EFMedia/audioTasks/makeAudioBufferTask.js +2 -1
  15. package/dist/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.js +2 -0
  16. package/dist/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.d.ts +1 -1
  17. package/dist/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.js +3 -1
  18. package/dist/elements/EFMedia/audioTasks/makeAudioInputTask.js +1 -1
  19. package/dist/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.d.ts +1 -1
  20. package/dist/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.js +6 -5
  21. package/dist/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.js +3 -1
  22. package/dist/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.js +2 -0
  23. package/dist/elements/EFMedia/shared/AudioSpanUtils.js +2 -2
  24. package/dist/elements/EFMedia/shared/GlobalInputCache.d.ts +39 -0
  25. package/dist/elements/EFMedia/shared/GlobalInputCache.js +57 -0
  26. package/dist/elements/EFMedia/shared/ThumbnailExtractor.d.ts +27 -0
  27. package/dist/elements/EFMedia/shared/ThumbnailExtractor.js +106 -0
  28. package/dist/elements/EFMedia/tasks/makeMediaEngineTask.js +1 -1
  29. package/dist/elements/EFMedia.d.ts +2 -2
  30. package/dist/elements/EFMedia.js +25 -1
  31. package/dist/elements/EFSurface.browsertest.d.ts +0 -0
  32. package/dist/elements/EFSurface.d.ts +30 -0
  33. package/dist/elements/EFSurface.js +96 -0
  34. package/dist/elements/EFTemporal.js +7 -6
  35. package/dist/elements/EFThumbnailStrip.browsertest.d.ts +0 -0
  36. package/dist/elements/EFThumbnailStrip.d.ts +86 -0
  37. package/dist/elements/EFThumbnailStrip.js +490 -0
  38. package/dist/elements/EFThumbnailStrip.media-engine.browsertest.d.ts +0 -0
  39. package/dist/elements/EFTimegroup.d.ts +6 -1
  40. package/dist/elements/EFTimegroup.js +53 -11
  41. package/dist/elements/updateAnimations.browsertest.d.ts +13 -0
  42. package/dist/elements/updateAnimations.d.ts +5 -0
  43. package/dist/elements/updateAnimations.js +37 -13
  44. package/dist/getRenderInfo.js +1 -1
  45. package/dist/gui/ContextMixin.js +27 -14
  46. package/dist/gui/EFControls.browsertest.d.ts +0 -0
  47. package/dist/gui/EFControls.d.ts +38 -0
  48. package/dist/gui/EFControls.js +51 -0
  49. package/dist/gui/EFFilmstrip.d.ts +40 -1
  50. package/dist/gui/EFFilmstrip.js +240 -3
  51. package/dist/gui/EFPreview.js +2 -1
  52. package/dist/gui/EFScrubber.d.ts +6 -5
  53. package/dist/gui/EFScrubber.js +31 -21
  54. package/dist/gui/EFTimeDisplay.browsertest.d.ts +0 -0
  55. package/dist/gui/EFTimeDisplay.d.ts +2 -6
  56. package/dist/gui/EFTimeDisplay.js +13 -23
  57. package/dist/gui/TWMixin.js +1 -1
  58. package/dist/gui/currentTimeContext.d.ts +3 -0
  59. package/dist/gui/currentTimeContext.js +3 -0
  60. package/dist/gui/durationContext.d.ts +3 -0
  61. package/dist/gui/durationContext.js +3 -0
  62. package/dist/index.d.ts +3 -0
  63. package/dist/index.js +4 -1
  64. package/dist/style.css +1 -1
  65. package/dist/transcoding/types/index.d.ts +11 -0
  66. package/dist/utils/LRUCache.d.ts +46 -0
  67. package/dist/utils/LRUCache.js +382 -1
  68. package/dist/utils/LRUCache.test.d.ts +1 -0
  69. package/package.json +2 -2
  70. package/src/elements/ContextProxiesController.ts +124 -0
  71. package/src/elements/EFCaptions.browsertest.ts +1820 -0
  72. package/src/elements/EFCaptions.ts +373 -36
  73. package/src/elements/EFImage.ts +4 -1
  74. package/src/elements/EFMedia/AssetIdMediaEngine.ts +30 -1
  75. package/src/elements/EFMedia/AssetMediaEngine.ts +33 -0
  76. package/src/elements/EFMedia/BaseMediaEngine.browsertest.ts +3 -8
  77. package/src/elements/EFMedia/BaseMediaEngine.ts +35 -0
  78. package/src/elements/EFMedia/JitMediaEngine.ts +34 -0
  79. package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.ts +6 -5
  80. package/src/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts +5 -0
  81. package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts +8 -5
  82. package/src/elements/EFMedia/audioTasks/makeAudioInputTask.ts +5 -5
  83. package/src/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.ts +11 -12
  84. package/src/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.ts +7 -4
  85. package/src/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts +5 -0
  86. package/src/elements/EFMedia/shared/AudioSpanUtils.ts +2 -2
  87. package/src/elements/EFMedia/shared/GlobalInputCache.ts +77 -0
  88. package/src/elements/EFMedia/shared/RenditionHelpers.browsertest.ts +2 -2
  89. package/src/elements/EFMedia/shared/RenditionHelpers.ts +2 -2
  90. package/src/elements/EFMedia/shared/ThumbnailExtractor.ts +227 -0
  91. package/src/elements/EFMedia/tasks/makeMediaEngineTask.ts +1 -1
  92. package/src/elements/EFMedia.ts +38 -1
  93. package/src/elements/EFSurface.browsertest.ts +155 -0
  94. package/src/elements/EFSurface.ts +141 -0
  95. package/src/elements/EFTemporal.ts +14 -8
  96. package/src/elements/EFThumbnailStrip.browsertest.ts +591 -0
  97. package/src/elements/EFThumbnailStrip.media-engine.browsertest.ts +713 -0
  98. package/src/elements/EFThumbnailStrip.ts +905 -0
  99. package/src/elements/EFTimegroup.browsertest.ts +56 -7
  100. package/src/elements/EFTimegroup.ts +88 -16
  101. package/src/elements/updateAnimations.browsertest.ts +333 -11
  102. package/src/elements/updateAnimations.ts +68 -19
  103. package/src/gui/ContextMixin.browsertest.ts +0 -25
  104. package/src/gui/ContextMixin.ts +44 -20
  105. package/src/gui/EFControls.browsertest.ts +175 -0
  106. package/src/gui/EFControls.ts +84 -0
  107. package/src/gui/EFFilmstrip.ts +323 -4
  108. package/src/gui/EFPreview.ts +2 -1
  109. package/src/gui/EFScrubber.ts +29 -25
  110. package/src/gui/EFTimeDisplay.browsertest.ts +237 -0
  111. package/src/gui/EFTimeDisplay.ts +12 -40
  112. package/src/gui/currentTimeContext.ts +5 -0
  113. package/src/gui/durationContext.ts +3 -0
  114. package/src/transcoding/types/index.ts +13 -0
  115. package/src/utils/LRUCache.test.ts +272 -0
  116. package/src/utils/LRUCache.ts +543 -0
  117. package/test/__cache__/GET__api_v1_transcode_high_1_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__26197f6f7c46cacb0a71134131c3f775/data.bin +0 -0
  118. package/test/__cache__/GET__api_v1_transcode_high_1_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__26197f6f7c46cacb0a71134131c3f775/metadata.json +1 -1
  119. package/test/__cache__/GET__api_v1_transcode_high_2_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__4cb6774cd3650ccf59c8f8dc6678c0b9/data.bin +0 -0
  120. package/test/__cache__/GET__api_v1_transcode_high_2_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__4cb6774cd3650ccf59c8f8dc6678c0b9/metadata.json +1 -1
  121. package/test/__cache__/GET__api_v1_transcode_high_3_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0b3b2b1c8933f7fcf8a9ecaa88d58b41/data.bin +0 -0
  122. package/test/__cache__/GET__api_v1_transcode_high_3_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0b3b2b1c8933f7fcf8a9ecaa88d58b41/metadata.json +1 -1
  123. package/test/__cache__/GET__api_v1_transcode_high_4_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a6fb05a22b18d850f7f2950bbcdbdeed/data.bin +0 -0
  124. package/test/__cache__/GET__api_v1_transcode_high_4_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a6fb05a22b18d850f7f2950bbcdbdeed/metadata.json +1 -1
  125. package/test/__cache__/GET__api_v1_transcode_high_5_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a50058c7c3602e90879fe3428ed891f4/data.bin +0 -0
  126. package/test/__cache__/GET__api_v1_transcode_high_5_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__a50058c7c3602e90879fe3428ed891f4/metadata.json +1 -1
  127. package/test/__cache__/GET__api_v1_transcode_high_init_m4s_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__0798c479b44aaeef850609a430f6e613/data.bin +0 -0
  128. package/test/__cache__/GET__api_v1_transcode_manifest_json_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__3be92a0437de726b431ed5af2369158a/data.bin +1 -1
  129. package/test/__cache__/GET__api_v1_transcode_manifest_json_url_http_3A_2F_2Fweb_3A3000_2Fhead_moov_480p_mp4__3be92a0437de726b431ed5af2369158a/metadata.json +1 -1
  130. package/types.json +1 -1
  131. package/dist/transcoding/cache/CacheManager.d.ts +0 -73
  132. package/src/transcoding/cache/CacheManager.ts +0 -208
@@ -7,6 +7,7 @@ import { BaseMediaEngine, mediaCache } from "./BaseMediaEngine.js";
7
7
  const test = baseTest.extend<{}>({});
8
8
 
9
9
  // Test implementation of BaseMediaEngine for testing
10
+ // @ts-expect-error missing implementations
10
11
  class TestMediaEngine extends BaseMediaEngine {
11
12
  fetchMediaSegment = vi.fn();
12
13
  public host: EFMedia;
@@ -221,14 +222,8 @@ describe("BaseMediaEngine abort signal handling", () => {
221
222
  const host = {
222
223
  fetch: vi
223
224
  .fn()
224
- .mockImplementation(
225
- () =>
226
- new Promise((resolve) =>
227
- setTimeout(
228
- () => resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
229
- 100,
230
- ),
231
- ),
225
+ .mockImplementation(() =>
226
+ Promise.resolve({ arrayBuffer: () => new ArrayBuffer(1024) }),
232
227
  ),
233
228
  } as any;
234
229
 
@@ -2,10 +2,12 @@ import { RequestDeduplicator } from "../../transcoding/cache/RequestDeduplicator
2
2
  import type {
3
3
  AudioRendition,
4
4
  SegmentTimeRange,
5
+ ThumbnailResult,
5
6
  VideoRendition,
6
7
  } from "../../transcoding/types";
7
8
  import { SizeAwareLRUCache } from "../../utils/LRUCache.js";
8
9
  import type { EFMedia } from "../EFMedia.js";
10
+ import type { MediaRendition } from "./shared/MediaTaskUtils.js";
9
11
 
10
12
  // Global instances shared across all media engines
11
13
  export const mediaCache = new SizeAwareLRUCache<string>(100 * 1024 * 1024); // 100MB cache limit
@@ -205,6 +207,16 @@ export abstract class BaseMediaEngine {
205
207
  rendition: { trackId: number | undefined; src: string },
206
208
  ): Promise<ArrayBuffer>;
207
209
 
210
+ abstract fetchInitSegment(
211
+ rendition: { trackId: number | undefined; src: string },
212
+ signal: AbortSignal,
213
+ ): Promise<ArrayBuffer>;
214
+
215
+ abstract computeSegmentId(
216
+ desiredSeekTimeMs: number,
217
+ rendition: MediaRendition,
218
+ ): number | undefined;
219
+
208
220
  /**
209
221
  * Fetch media segment with built-in deduplication
210
222
  * Now uses global deduplication for all requests
@@ -387,4 +399,27 @@ export abstract class BaseMediaEngine {
387
399
  segmentIds.filter((id) => this.isSegmentCached(id, rendition)),
388
400
  );
389
401
  }
402
+
403
+ /**
404
+ * Extract thumbnail canvases at multiple timestamps efficiently
405
+ * Default implementation provides helpful error information
406
+ */
407
+ async extractThumbnails(
408
+ timestamps: number[],
409
+ ): Promise<(ThumbnailResult | null)[]> {
410
+ const engineName = this.constructor.name;
411
+ console.warn(
412
+ `${engineName}: extractThumbnails not properly implemented. ` +
413
+ "This MediaEngine type does not support thumbnail generation. " +
414
+ "Supported engines: JitMediaEngine. " +
415
+ `Requested ${timestamps.length} thumbnail${timestamps.length === 1 ? "" : "s"}.`,
416
+ );
417
+ return timestamps.map(() => null);
418
+ }
419
+
420
+ abstract convertToSegmentRelativeTimestamps(
421
+ globalTimestamps: number[],
422
+ segmentId: number,
423
+ rendition: VideoRendition,
424
+ ): number[];
390
425
  }
@@ -2,16 +2,19 @@ import type {
2
2
  AudioRendition,
3
3
  MediaEngine,
4
4
  RenditionId,
5
+ ThumbnailResult,
5
6
  VideoRendition,
6
7
  } from "../../transcoding/types";
7
8
  import type { ManifestResponse } from "../../transcoding/types/index.js";
8
9
  import type { UrlGenerator } from "../../transcoding/utils/UrlGenerator";
9
10
  import type { EFMedia } from "../EFMedia.js";
10
11
  import { BaseMediaEngine } from "./BaseMediaEngine";
12
+ import { ThumbnailExtractor } from "./shared/ThumbnailExtractor.js";
11
13
 
12
14
  export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
13
15
  private urlGenerator: UrlGenerator;
14
16
  private data: ManifestResponse = {} as ManifestResponse;
17
+ private thumbnailExtractor: ThumbnailExtractor;
15
18
 
16
19
  static async fetch(host: EFMedia, urlGenerator: UrlGenerator, url: string) {
17
20
  const engine = new JitMediaEngine(host, urlGenerator);
@@ -23,6 +26,7 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
23
26
  constructor(host: EFMedia, urlGenerator: UrlGenerator) {
24
27
  super(host);
25
28
  this.urlGenerator = urlGenerator;
29
+ this.thumbnailExtractor = new ThumbnailExtractor(this);
26
30
  }
27
31
 
28
32
  get durationMs() {
@@ -204,4 +208,34 @@ export class JitMediaEngine extends BaseMediaEngine implements MediaEngine {
204
208
  maxAudioBufferFetches: 3,
205
209
  };
206
210
  }
211
+
212
+ /**
213
+ * Extract thumbnail canvases using same rendition priority as video playback for frame alignment
214
+ */
215
+ async extractThumbnails(
216
+ timestamps: number[],
217
+ ): Promise<(ThumbnailResult | null)[]> {
218
+ const mainRendition = this.videoRendition;
219
+ const scrubRendition = this.getScrubVideoRendition();
220
+
221
+ const rendition = mainRendition || scrubRendition;
222
+
223
+ if (!rendition) {
224
+ return timestamps.map(() => null);
225
+ }
226
+
227
+ return this.thumbnailExtractor.extractThumbnails(
228
+ timestamps,
229
+ rendition,
230
+ this.durationMs,
231
+ );
232
+ }
233
+
234
+ convertToSegmentRelativeTimestamps(
235
+ globalTimestamps: number[],
236
+ _segmentId: number,
237
+ _rendition: VideoRendition,
238
+ ): number[] {
239
+ return globalTimestamps.map((timestamp) => timestamp / 1000);
240
+ }
207
241
  }
@@ -42,13 +42,15 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
42
42
  task: async ([seekTimeMs], { signal }) => {
43
43
  // Skip buffering entirely in rendering mode
44
44
  if (EF_RENDERING()) {
45
- return currentState; // Return existing state without any buffering activity
45
+ return currentState;
46
46
  }
47
47
 
48
- // Get media engine to potentially override buffer configuration
49
48
  const mediaEngine = await getLatestMediaEngine(host, signal);
50
49
 
51
- // Use media engine's buffer config, falling back to host properties
50
+ if (!mediaEngine.audioRendition) {
51
+ return currentState;
52
+ }
53
+
52
54
  const engineConfig = mediaEngine.getBufferConfig();
53
55
  const bufferDurationMs = engineConfig.audioBufferDurationMs;
54
56
  const maxParallelFetches = engineConfig.maxAudioBufferFetches;
@@ -85,11 +87,10 @@ export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
85
87
  return mediaEngine.isSegmentCached(segmentId, rendition);
86
88
  },
87
89
  getRendition: async () => {
88
- // Get real audio rendition from media engine
89
90
  const mediaEngine = await getLatestMediaEngine(host, signal);
90
91
  const audioRendition = mediaEngine.audioRendition;
91
92
  if (!audioRendition) {
92
- throw new Error("Audio rendition not available");
93
+ throw new Error("No audio track available in source");
93
94
  }
94
95
  return audioRendition;
95
96
  },
@@ -95,6 +95,11 @@ export function makeAudioFrequencyAnalysisTask(element: EFMedia) {
95
95
  task: async (_, { signal }) => {
96
96
  if (element.currentSourceTimeMs < 0) return null;
97
97
 
98
+ const mediaEngine = element.mediaEngineTask.value;
99
+ if (!mediaEngine?.audioRendition) {
100
+ return null;
101
+ }
102
+
98
103
  const currentTimeMs = element.currentSourceTimeMs;
99
104
 
100
105
  // Calculate exact audio window needed based on fftDecay and frame timing
@@ -5,7 +5,7 @@ import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
5
5
 
6
6
  export const makeAudioInitSegmentFetchTask = (
7
7
  host: EFMedia,
8
- ): Task<readonly [MediaEngine | undefined], ArrayBuffer> => {
8
+ ): Task<readonly [MediaEngine | undefined], ArrayBuffer | undefined> => {
9
9
  return new Task(host, {
10
10
  args: () => [host.mediaEngineTask.value] as const,
11
11
  onError: (error) => {
@@ -14,10 +14,13 @@ export const makeAudioInitSegmentFetchTask = (
14
14
  onComplete: (_value) => {},
15
15
  task: async ([_mediaEngine], { signal }) => {
16
16
  const mediaEngine = await getLatestMediaEngine(host, signal);
17
- return mediaEngine.fetchInitSegment(
18
- mediaEngine.getAudioRendition(),
19
- signal,
20
- );
17
+
18
+ const audioRendition = mediaEngine.audioRendition;
19
+ if (!audioRendition) {
20
+ return undefined;
21
+ }
22
+
23
+ return mediaEngine.fetchInitSegment(audioRendition, signal);
21
24
  },
22
25
  });
23
26
  };
@@ -19,20 +19,20 @@ export const makeAudioInputTask = (host: EFMedia): InputTask => {
19
19
  onComplete: (_value) => {},
20
20
  task: async (_, { signal }) => {
21
21
  const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
22
- signal.throwIfAborted(); // Abort if a new seek started
22
+ signal.throwIfAborted();
23
23
  const segment = await host.audioSegmentFetchTask.taskComplete;
24
- signal.throwIfAborted(); // Abort if a new seek started
24
+ signal.throwIfAborted();
25
+
25
26
  if (!initSegment || !segment) {
26
- throw new Error("Init segment or segment is not available");
27
+ throw new Error("No audio track available in source");
27
28
  }
28
29
 
29
- // Get startTimeOffsetMs from the audio rendition if available
30
30
  const mediaEngine = await host.mediaEngineTask.taskComplete;
31
31
  const audioRendition = mediaEngine?.audioRendition;
32
32
  const startTimeOffsetMs = audioRendition?.startTimeOffsetMs;
33
33
 
34
34
  const arrayBuffer = await new Blob([initSegment, segment]).arrayBuffer();
35
- signal.throwIfAborted(); // Abort if a new seek started
35
+ signal.throwIfAborted();
36
36
  return new BufferedSeekingInput(arrayBuffer, {
37
37
  videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
38
38
  audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
@@ -7,7 +7,7 @@ export const makeAudioSegmentFetchTask = (
7
7
  host: EFMedia,
8
8
  ): Task<
9
9
  readonly [MediaEngine | undefined, number | undefined],
10
- ArrayBuffer
10
+ ArrayBuffer | undefined
11
11
  > => {
12
12
  return new Task(host, {
13
13
  args: () =>
@@ -18,14 +18,18 @@ export const makeAudioSegmentFetchTask = (
18
18
  onComplete: (_value) => {},
19
19
  task: async (_, { signal }) => {
20
20
  const mediaEngine = await getLatestMediaEngine(host, signal);
21
+
22
+ const audioRendition = mediaEngine.audioRendition;
23
+ if (!audioRendition) {
24
+ return undefined;
25
+ }
26
+
21
27
  const segmentId = await host.audioSegmentIdTask.taskComplete;
22
28
  if (segmentId === undefined) {
23
- // Provide more context in the error to help with debugging
24
- const rendition = mediaEngine.audioRendition;
25
29
  const debugInfo = {
26
- hasRendition: !!rendition,
27
- segmentDurationMs: rendition?.segmentDurationMs,
28
- segmentDurationsMs: rendition?.segmentDurationsMs?.length || 0,
30
+ hasRendition: true,
31
+ segmentDurationMs: audioRendition.segmentDurationMs,
32
+ segmentDurationsMs: audioRendition.segmentDurationsMs?.length || 0,
29
33
  desiredSeekTimeMs: host.desiredSeekTimeMs,
30
34
  intrinsicDurationMs: host.intrinsicDurationMs,
31
35
  };
@@ -34,12 +38,7 @@ export const makeAudioSegmentFetchTask = (
34
38
  );
35
39
  }
36
40
 
37
- // SIMPLIFIED: Direct call to mediaEngine - deduplication is built-in
38
- return mediaEngine.fetchMediaSegment(
39
- segmentId,
40
- mediaEngine.getAudioRendition(),
41
- signal,
42
- );
41
+ return mediaEngine.fetchMediaSegment(segmentId, audioRendition, signal);
43
42
  },
44
43
  });
45
44
  };
@@ -15,10 +15,13 @@ export const makeAudioSegmentIdTask = (
15
15
  task: async ([, targetSeekTimeMs], { signal }) => {
16
16
  const mediaEngine = await getLatestMediaEngine(host, signal);
17
17
  signal.throwIfAborted(); // Abort if a new seek started
18
- return mediaEngine.computeSegmentId(
19
- targetSeekTimeMs, // Use captured value, not host.desiredSeekTimeMs
20
- mediaEngine.getAudioRendition(),
21
- );
18
+
19
+ const audioRendition = mediaEngine.audioRendition;
20
+ if (!audioRendition) {
21
+ return undefined;
22
+ }
23
+
24
+ return mediaEngine.computeSegmentId(targetSeekTimeMs, audioRendition);
22
25
  },
23
26
  });
24
27
  };
@@ -27,6 +27,11 @@ export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
27
27
  task: async (_, { signal }) => {
28
28
  if (element.currentSourceTimeMs < 0) return null;
29
29
 
30
+ const mediaEngine = element.mediaEngineTask.value;
31
+ if (!mediaEngine?.audioRendition) {
32
+ return null;
33
+ }
34
+
30
35
  const currentTimeMs = element.currentSourceTimeMs;
31
36
 
32
37
  // Calculate exact audio window needed based on fftDecay and frame timing
@@ -16,7 +16,7 @@ const fetchAudioSegmentData = async (
16
16
  ): Promise<Map<number, ArrayBuffer>> => {
17
17
  const audioRendition = mediaEngine.audioRendition;
18
18
  if (!audioRendition) {
19
- throw new Error("Audio rendition not available");
19
+ throw new Error("No audio track available in source");
20
20
  }
21
21
 
22
22
  const segmentData = new Map<number, ArrayBuffer>();
@@ -73,7 +73,7 @@ export const fetchAudioSpanningTime = async (
73
73
  const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
74
74
 
75
75
  if (!mediaEngine?.audioRendition) {
76
- throw new Error("Audio rendition not available");
76
+ throw new Error("No audio track available in source");
77
77
  }
78
78
 
79
79
  if (!initSegment) {
@@ -0,0 +1,77 @@
1
+ import type { Input } from "mediabunny";
2
+ import { LRUCache } from "../../../utils/LRUCache.js";
3
+
4
+ /**
5
+ * Global cache for MediaBunny Input instances
6
+ * Shared across all MediaEngine instances to prevent duplicate decoding
7
+ * of the same segment data
8
+ */
9
+ class GlobalInputCache {
10
+ private cache = new LRUCache<string, Input>(50); // 50 Input instances max
11
+
12
+ /**
13
+ * Generate standardized cache key for Input objects
14
+ * Format: "input:{src}:{segmentId}:{renditionId}"
15
+ */
16
+ private generateKey(
17
+ src: string,
18
+ segmentId: number,
19
+ renditionId?: string,
20
+ ): string {
21
+ return `input:${src}:${segmentId}:${renditionId || "default"}`;
22
+ }
23
+
24
+ /**
25
+ * Get cached Input object
26
+ */
27
+ get(src: string, segmentId: number, renditionId?: string): Input | undefined {
28
+ const key = this.generateKey(src, segmentId, renditionId);
29
+ return this.cache.get(key);
30
+ }
31
+
32
+ /**
33
+ * Cache Input object
34
+ */
35
+ set(
36
+ src: string,
37
+ segmentId: number,
38
+ input: Input,
39
+ renditionId?: string,
40
+ ): void {
41
+ const key = this.generateKey(src, segmentId, renditionId);
42
+ this.cache.set(key, input);
43
+ }
44
+
45
+ /**
46
+ * Check if Input is cached
47
+ */
48
+ has(src: string, segmentId: number, renditionId?: string): boolean {
49
+ const key = this.generateKey(src, segmentId, renditionId);
50
+ return this.cache.has(key);
51
+ }
52
+
53
+ /**
54
+ * Clear all cached Input objects
55
+ */
56
+ clear(): void {
57
+ this.cache.clear();
58
+ }
59
+
60
+ /**
61
+ * Get cache statistics for debugging
62
+ */
63
+ getStats() {
64
+ return {
65
+ size: this.cache.size,
66
+ cachedKeys: Array.from((this.cache as any).cache.keys()),
67
+ };
68
+ }
69
+ }
70
+
71
+ // Single global instance shared across all MediaEngine instances
72
+ export const globalInputCache = new GlobalInputCache();
73
+
74
+ // Export for debugging (works in both browser and server)
75
+ (
76
+ globalThis as typeof globalThis & { debugInputCache: typeof globalInputCache }
77
+ ).debugInputCache = globalInputCache;
@@ -107,7 +107,7 @@ describe("RenditionHelpers", () => {
107
107
  expect,
108
108
  }) => {
109
109
  expect(() => getAudioRendition(mockMediaEngineWithoutAudio)).toThrow(
110
- "Audio rendition is not available",
110
+ "No audio track available in source",
111
111
  );
112
112
  });
113
113
  });
@@ -128,7 +128,7 @@ describe("RenditionHelpers", () => {
128
128
  expect,
129
129
  }) => {
130
130
  expect(() => getVideoRendition(mockMediaEngineWithoutVideo)).toThrow(
131
- "Video rendition is not available",
131
+ "No video track available in source",
132
132
  );
133
133
  });
134
134
  });
@@ -10,7 +10,7 @@ import type {
10
10
  export const getAudioRendition = (mediaEngine: MediaEngine): AudioRendition => {
11
11
  const audioRendition = mediaEngine.audioRendition;
12
12
  if (!audioRendition) {
13
- throw new Error("Audio rendition is not available");
13
+ throw new Error("No audio track available in source");
14
14
  }
15
15
  return audioRendition;
16
16
  };
@@ -21,7 +21,7 @@ export const getAudioRendition = (mediaEngine: MediaEngine): AudioRendition => {
21
21
  export const getVideoRendition = (mediaEngine: MediaEngine): VideoRendition => {
22
22
  const videoRendition = mediaEngine.videoRendition;
23
23
  if (!videoRendition) {
24
- throw new Error("Video rendition is not available");
24
+ throw new Error("No video track available in source");
25
25
  }
26
26
  return videoRendition;
27
27
  };
@@ -0,0 +1,227 @@
1
+ import { ALL_FORMATS, BlobSource, CanvasSink, Input } from "mediabunny";
2
+ import type {
3
+ ThumbnailResult,
4
+ VideoRendition,
5
+ } from "../../../transcoding/types/index.js";
6
+ import type { BaseMediaEngine } from "../BaseMediaEngine.js";
7
+ import { globalInputCache } from "./GlobalInputCache.js";
8
+
9
+ /**
10
+ * Shared thumbnail extraction logic for all MediaEngine implementations
11
+ * Eliminates code duplication and provides consistent behavior
12
+ */
13
+ export class ThumbnailExtractor {
14
+ constructor(private mediaEngine: BaseMediaEngine) {}
15
+
16
+ /**
17
+ * Extract thumbnails at multiple timestamps efficiently using segment batching
18
+ */
19
+ async extractThumbnails(
20
+ timestamps: number[],
21
+ rendition: VideoRendition,
22
+ durationMs: number,
23
+ ): Promise<(ThumbnailResult | null)[]> {
24
+ if (timestamps.length === 0) {
25
+ return [];
26
+ }
27
+
28
+ // Validate and filter timestamps within bounds
29
+ const validTimestamps = timestamps.filter(
30
+ (timeMs) => timeMs >= 0 && timeMs <= durationMs,
31
+ );
32
+
33
+ if (validTimestamps.length === 0) {
34
+ console.warn(
35
+ `ThumbnailExtractor: All timestamps out of bounds (0-${durationMs}ms)`,
36
+ );
37
+ return timestamps.map(() => null);
38
+ }
39
+
40
+ // Group timestamps by segment for batch processing
41
+ const segmentGroups = this.groupTimestampsBySegment(
42
+ validTimestamps,
43
+ rendition,
44
+ );
45
+
46
+ // Extract batched by segment using CanvasSink
47
+ const results = new Map<number, ThumbnailResult | null>();
48
+
49
+ for (const [segmentId, segmentTimestamps] of segmentGroups) {
50
+ try {
51
+ const segmentResults = await this.extractSegmentThumbnails(
52
+ segmentId,
53
+ segmentTimestamps,
54
+ rendition,
55
+ );
56
+
57
+ for (const [timestamp, thumbnail] of segmentResults) {
58
+ results.set(timestamp, thumbnail);
59
+ }
60
+ } catch (error) {
61
+ console.warn(
62
+ `ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
63
+ error,
64
+ );
65
+ // Mark all timestamps in this segment as failed
66
+ for (const timestamp of segmentTimestamps) {
67
+ results.set(timestamp, null);
68
+ }
69
+ }
70
+ }
71
+
72
+ // Return in original order, null for any that failed or were out of bounds
73
+ return timestamps.map((t) => {
74
+ // If timestamp was out of bounds, return null
75
+ if (t < 0 || t > durationMs) {
76
+ return null;
77
+ }
78
+ return results.get(t) || null;
79
+ });
80
+ }
81
+
82
+ /**
83
+ * Group timestamps by segment ID for efficient batch processing
84
+ */
85
+ private groupTimestampsBySegment(
86
+ timestamps: number[],
87
+ rendition: VideoRendition,
88
+ ): Map<number, number[]> {
89
+ const segmentGroups = new Map<number, number[]>();
90
+
91
+ for (const timeMs of timestamps) {
92
+ try {
93
+ const segmentId = this.mediaEngine.computeSegmentId(timeMs, rendition);
94
+ if (segmentId !== undefined) {
95
+ if (!segmentGroups.has(segmentId)) {
96
+ segmentGroups.set(segmentId, []);
97
+ }
98
+ const segmentGroup = segmentGroups.get(segmentId) ?? [];
99
+ if (!segmentGroup) {
100
+ segmentGroups.set(segmentId, []);
101
+ }
102
+ segmentGroup.push(timeMs);
103
+ }
104
+ } catch (error) {
105
+ console.warn(
106
+ `ThumbnailExtractor: Could not compute segment for timestamp ${timeMs}:`,
107
+ error,
108
+ );
109
+ }
110
+ }
111
+
112
+ return segmentGroups;
113
+ }
114
+
115
+ /**
116
+ * Extract thumbnails for a specific segment using CanvasSink
117
+ */
118
+ private async extractSegmentThumbnails(
119
+ segmentId: number,
120
+ timestamps: number[],
121
+ rendition: VideoRendition,
122
+ ): Promise<Map<number, ThumbnailResult | null>> {
123
+ const results = new Map<number, ThumbnailResult | null>();
124
+
125
+ try {
126
+ // Get segment data through existing media engine methods (uses caches)
127
+ const abortController = new AbortController();
128
+ const [initSegment, mediaSegment] = await Promise.all([
129
+ this.mediaEngine.fetchInitSegment(rendition, abortController.signal),
130
+ this.mediaEngine.fetchMediaSegment(segmentId, rendition),
131
+ ]);
132
+
133
+ // Create Input for this segment using global shared cache
134
+ const segmentBlob = new Blob([initSegment, mediaSegment]);
135
+
136
+ let input = globalInputCache.get(rendition.src, segmentId, rendition.id);
137
+ if (!input) {
138
+ input = new Input({
139
+ formats: ALL_FORMATS,
140
+ source: new BlobSource(segmentBlob),
141
+ });
142
+ globalInputCache.set(rendition.src, segmentId, input, rendition.id);
143
+ }
144
+
145
+ // Set up CanvasSink for batched extraction
146
+ const videoTrack = await input.getPrimaryVideoTrack();
147
+ if (!videoTrack) {
148
+ // No video track - return nulls for all timestamps
149
+ for (const timestamp of timestamps) {
150
+ results.set(timestamp, null);
151
+ }
152
+ return results;
153
+ }
154
+
155
+ const sink = new CanvasSink(videoTrack);
156
+
157
+ // Convert global timestamps to segment-relative (in seconds for mediabunny)
158
+ const relativeTimestamps = this.convertToSegmentRelativeTimestamps(
159
+ timestamps,
160
+ segmentId,
161
+ rendition,
162
+ );
163
+
164
+ // Batch extract all thumbnails for this segment
165
+ const timestampResults = [];
166
+ for await (const result of sink.canvasesAtTimestamps(
167
+ relativeTimestamps,
168
+ )) {
169
+ timestampResults.push(result);
170
+ }
171
+
172
+ // Map results back to original timestamps
173
+ for (let i = 0; i < timestamps.length; i++) {
174
+ const globalTimestamp = timestamps[i];
175
+ if (globalTimestamp === undefined) {
176
+ continue;
177
+ }
178
+
179
+ const result = timestampResults[i];
180
+
181
+ if (result?.canvas) {
182
+ const canvas = result.canvas;
183
+ if (
184
+ canvas instanceof HTMLCanvasElement ||
185
+ canvas instanceof OffscreenCanvas
186
+ ) {
187
+ results.set(globalTimestamp, {
188
+ timestamp: globalTimestamp,
189
+ thumbnail: canvas,
190
+ });
191
+ } else {
192
+ results.set(globalTimestamp, null);
193
+ }
194
+ } else {
195
+ results.set(globalTimestamp, null);
196
+ }
197
+ }
198
+ } catch (error) {
199
+ console.error(
200
+ `ThumbnailExtractor: Failed to extract thumbnails for segment ${segmentId}:`,
201
+ error,
202
+ );
203
+ // Return nulls for all timestamps on error
204
+ for (const timestamp of timestamps) {
205
+ results.set(timestamp, null);
206
+ }
207
+ }
208
+
209
+ return results;
210
+ }
211
+
212
+ /**
213
+ * Convert global timestamps to segment-relative timestamps for mediabunny
214
+ * This is where the main difference between JIT and Asset engines lies
215
+ */
216
+ private convertToSegmentRelativeTimestamps(
217
+ globalTimestamps: number[],
218
+ segmentId: number,
219
+ rendition: VideoRendition,
220
+ ): number[] {
221
+ return this.mediaEngine.convertToSegmentRelativeTimestamps(
222
+ globalTimestamps,
223
+ segmentId,
224
+ rendition,
225
+ );
226
+ }
227
+ }