@editframe/elements 0.17.6-beta.0 → 0.18.7-beta.0

package/src/elements/EFMedia/audioTasks/makeAudioSeekTask.ts

@@ -0,0 +1,64 @@
+import { Task } from "@lit/task";
+import type { VideoSample } from "mediabunny";
+import { type EFMedia, IgnorableError } from "../../EFMedia";
+import type { BufferedSeekingInput } from "../BufferedSeekingInput";
+
+type AudioSeekTask = Task<
+  readonly [number, BufferedSeekingInput | undefined],
+  VideoSample | undefined
+>;
+export const makeAudioSeekTask = (host: EFMedia): AudioSeekTask => {
+  return new Task(host, {
+    args: () => [host.desiredSeekTimeMs, host.audioInputTask.value] as const,
+    onError: (error) => {
+      if (error instanceof IgnorableError) {
+        console.info("audioSeekTask aborted");
+      }
+      console.error("audioSeekTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async (
+      [targetSeekTimeMs],
+      { signal },
+    ): Promise<VideoSample | undefined> => {
+      // CRITICAL FIX: Use the targetSeekTimeMs from args, not host.desiredSeekTimeMs
+      // This ensures we use the same seek time that the segment loading tasks used
+
+      await host.audioSegmentIdTask.taskComplete;
+      signal.throwIfAborted(); // Abort if a new seek started
+      await host.audioSegmentFetchTask.taskComplete;
+      signal.throwIfAborted(); // Abort if a new seek started
+      await host.audioInitSegmentFetchTask.taskComplete;
+      signal.throwIfAborted(); // Abort if a new seek started
+
+      const audioInput = await host.audioInputTask.taskComplete;
+      signal.throwIfAborted(); // Abort if a new seek started
+      if (!audioInput) {
+        throw new Error("Audio input is not available");
+      }
+      const audioTrack = await audioInput.getFirstAudioTrack();
+      if (!audioTrack) {
+        throw new Error("Audio track is not available");
+      }
+      signal.throwIfAborted(); // Abort if a new seek started
+
+      const sample = (await audioInput.seek(
+        audioTrack.id,
+        targetSeekTimeMs, // Use the captured value, not host.desiredSeekTimeMs
+      )) as unknown as VideoSample | undefined;
+      signal.throwIfAborted(); // Abort if a new seek started
+
+      // If seek returned undefined, it was aborted - don't throw
+      if (sample === undefined && signal.aborted) {
+        return undefined;
+      }
+
+      // If we got undefined but weren't aborted, that's an actual error
+      if (sample === undefined) {
+        throw new Error("Audio seek failed to find sample");
+      }
+
+      return sample;
+    },
+  });
+};

package/src/elements/EFMedia/audioTasks/makeAudioSegmentFetchTask.ts

@@ -0,0 +1,45 @@
+import { Task } from "@lit/task";
+import type { MediaEngine } from "../../../transcoding/types";
+import type { EFMedia } from "../../EFMedia";
+import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
+
+export const makeAudioSegmentFetchTask = (
+  host: EFMedia,
+): Task<
+  readonly [MediaEngine | undefined, number | undefined],
+  ArrayBuffer
+> => {
+  return new Task(host, {
+    args: () =>
+      [host.mediaEngineTask.value, host.audioSegmentIdTask.value] as const,
+    onError: (error) => {
+      console.error("audioSegmentFetchTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async (_, { signal }) => {
+      const mediaEngine = await getLatestMediaEngine(host, signal);
+      const segmentId = await host.audioSegmentIdTask.taskComplete;
+      if (segmentId === undefined) {
+        // Provide more context in the error to help with debugging
+        const rendition = mediaEngine.audioRendition;
+        const debugInfo = {
+          hasRendition: !!rendition,
+          segmentDurationMs: rendition?.segmentDurationMs,
+          segmentDurationsMs: rendition?.segmentDurationsMs?.length || 0,
+          desiredSeekTimeMs: host.desiredSeekTimeMs,
+          intrinsicDurationMs: host.intrinsicDurationMs,
+        };
+        throw new Error(
+          `Segment ID is not available for audio. Debug info: ${JSON.stringify(debugInfo)}`,
+        );
+      }
+
+      // SIMPLIFIED: Direct call to mediaEngine - deduplication is built-in
+      return mediaEngine.fetchMediaSegment(
+        segmentId,
+        mediaEngine.getAudioRendition(),
+        signal,
+      );
+    },
+  });
+};

package/src/elements/EFMedia/audioTasks/makeAudioSegmentIdTask.ts

@@ -0,0 +1,24 @@
+import { Task } from "@lit/task";
+import type { MediaEngine } from "../../../transcoding/types";
+import type { EFMedia } from "../../EFMedia";
+import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
+
+export const makeAudioSegmentIdTask = (
+  host: EFMedia,
+): Task<readonly [MediaEngine | undefined, number], number | undefined> => {
+  return new Task(host, {
+    args: () => [host.mediaEngineTask.value, host.desiredSeekTimeMs] as const,
+    onError: (error) => {
+      console.error("audioSegmentIdTask error", error);
+    },
+    onComplete: (_value) => {},
+    task: async ([, targetSeekTimeMs], { signal }) => {
+      const mediaEngine = await getLatestMediaEngine(host, signal);
+      signal.throwIfAborted(); // Abort if a new seek started
+      return mediaEngine.computeSegmentId(
+        targetSeekTimeMs, // Use captured value, not host.desiredSeekTimeMs
+        mediaEngine.getAudioRendition(),
+      );
+    },
+  });
+};

package/src/elements/EFMedia/audioTasks/makeAudioTimeDomainAnalysisTask.ts

@@ -0,0 +1,183 @@
+import { Task } from "@lit/task";
+
+import { EF_INTERACTIVE } from "../../../EF_INTERACTIVE.js";
+import { LRUCache } from "../../../utils/LRUCache.js";
+import type { EFMedia } from "../../EFMedia.js";
+
+// DECAY_WEIGHT constant - same as original
+const DECAY_WEIGHT = 0.8;
+
+export function makeAudioTimeDomainAnalysisTask(element: EFMedia) {
+  // Internal cache for this task instance (same as original #byteTimeDomainCache)
+  const cache = new LRUCache<string, Uint8Array>(1000);
+
+  return new Task(element, {
+    autoRun: EF_INTERACTIVE,
+    onError: (error) => {
+      console.error("byteTimeDomainTask error", error);
+    },
+    args: () =>
+      [
+        element.audioBufferTask.status,
+        element.currentSourceTimeMs,
+        element.fftSize,
+        element.fftDecay,
+        element.fftGain,
+        element.shouldInterpolateFrequencies,
+      ] as const,
+    task: async () => {
+      await element.audioBufferTask.taskComplete;
+      if (!element.audioBufferTask.value) return null;
+      if (element.currentSourceTimeMs < 0) return null;
+
+      const currentTimeMs = element.currentSourceTimeMs;
+
+      // ONLY CHANGE: Get real audio data for analysis (same technique as playback)
+      const analysisWindowMs = 5000; // Get 5 seconds for better analysis
+      const fromMs = Math.max(0, currentTimeMs);
+      // Clamp toMs to video duration to prevent requesting segments beyond available content
+      const maxToMs = fromMs + analysisWindowMs;
+      const videoDurationMs = element.intrinsicDurationMs || 0;
+      const toMs =
+        videoDurationMs > 0 ? Math.min(maxToMs, videoDurationMs) : maxToMs;
+
+      // If the clamping results in an invalid range (seeking beyond the end), skip analysis silently
+      if (fromMs >= toMs) {
+        return null;
+      }
+
+      const { fetchAudioSpanningTime: fetchAudioSpan } = await import(
+        "../shared/AudioSpanUtils.ts"
+      );
+      const audioSpan = await fetchAudioSpan(
+        element,
+        fromMs,
+        toMs,
+        new AbortController().signal,
+      );
+
+      if (!audioSpan || !audioSpan.blob) {
+        console.warn("Time domain analysis skipped: no audio data available");
+        return null;
+      }
+
+      // Decode the real audio data
+      const tempAudioContext = new OfflineAudioContext(2, 48000, 48000);
+      const arrayBuffer = await audioSpan.blob.arrayBuffer();
+      const audioBuffer = await tempAudioContext.decodeAudioData(arrayBuffer);
+
+      // Use actual startOffset from audioSpan (relative to requested time)
+      const startOffsetMs = audioSpan.startMs;
+
+      // ORIGINAL ALGORITHM FROM HERE - unchanged customer logic
+      const smoothedKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftDecay}:${element.fftGain}:${startOffsetMs}:${currentTimeMs}`;
+      const cachedData = cache.get(smoothedKey);
+      if (cachedData) {
+        return cachedData;
+      }
+
+      // Process multiple frames with decay, similar to the reference code
+      const framesData = await Promise.all(
+        Array.from({ length: element.fftDecay }, async (_, frameIndex) => {
+          const frameOffset = frameIndex * (1000 / 30);
+          const startTime = Math.max(
+            0,
+            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
+          );
+
+          const cacheKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftGain}:${startOffsetMs}:${startTime}`;
+          const cachedFrame = cache.get(cacheKey);
+          if (cachedFrame) {
+            return cachedFrame;
+          }
+
+          let audioContext: OfflineAudioContext;
+          try {
+            audioContext = new OfflineAudioContext(2, 48000 * (1 / 30), 48000);
+          } catch (error) {
+            throw new Error(
+              `[EFMedia.byteTimeDomainTask] Failed to create OfflineAudioContext(2, ${48000 * (1 / 30)}, 48000) for frame ${frameIndex} at time ${startTime}s: ${error instanceof Error ? error.message : String(error)}. This is for audio time domain analysis.`,
+            );
+          }
+
+          const source = audioContext.createBufferSource();
+          source.buffer = audioBuffer;
+
+          // Create analyzer for PCM data
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = element.fftSize; // Ensure enough samples
+          analyser.minDecibels = -90;
+          analyser.maxDecibels = -20;
+
+          const gainNode = audioContext.createGain();
+          gainNode.gain.value = element.fftGain; // Amplify the signal
+
+          source.connect(gainNode);
+          gainNode.connect(analyser);
+          analyser.connect(audioContext.destination);
+
+          source.start(0, startTime, 1 / 30);
+
+          const dataLength = analyser.fftSize / 2;
+          try {
+            await audioContext.startRendering();
+            const frameData = new Uint8Array(dataLength);
+            analyser.getByteTimeDomainData(frameData);
+
+            // const points = frameData;
+            // Calculate RMS and midpoint values
+            const points = new Uint8Array(dataLength);
+            for (let i = 0; i < dataLength; i++) {
+              const pointSamples = frameData.slice(
+                i * (frameData.length / dataLength),
+                (i + 1) * (frameData.length / dataLength),
+              );
+
+              // Calculate RMS while preserving sign
+              const rms = Math.sqrt(
+                pointSamples.reduce((sum, sample) => {
+                  const normalized = (sample - 128) / 128;
+                  return sum + normalized * normalized;
+                }, 0) / pointSamples.length,
+              );
+
+              // Get average sign of the samples to determine direction
+              const avgSign = Math.sign(
+                pointSamples.reduce((sum, sample) => sum + (sample - 128), 0),
+              );
+
+              // Convert RMS back to byte range, preserving direction
+              points[i] = Math.min(255, Math.round(128 + avgSign * rms * 128));
+            }
+
+            cache.set(cacheKey, points);
+            return points;
+          } finally {
+            source.disconnect();
+            analyser.disconnect();
+          }
+        }),
+      );
+
+      // Combine frames with decay weighting
+      const frameLength = framesData[0]?.length ?? 0;
+      const smoothedData = new Uint8Array(frameLength);
+
+      for (let i = 0; i < frameLength; i++) {
+        let weightedSum = 0;
+        let weightSum = 0;
+
+        framesData.forEach((frame: Uint8Array, frameIndex: number) => {
+          const decayWeight = DECAY_WEIGHT ** frameIndex;
+          weightedSum += (frame[i] ?? 0) * decayWeight;
+          weightSum += decayWeight;
+        });
+
+        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
+      }
+
+      cache.set(smoothedKey, smoothedData);
+      return smoothedData;
+    },
+  });
+}

package/src/elements/EFMedia/shared/AudioSpanUtils.ts

@@ -0,0 +1,128 @@
+import type {
+  AudioSpan,
+  MediaEngine,
+  SegmentTimeRange,
+} from "../../../transcoding/types";
+import type { EFMedia } from "../../EFMedia";
+
+/**
+ * Fetch audio segment data using MediaEngine
+ * Pure function with explicit dependencies
+ */
+const fetchAudioSegmentData = async (
+  segmentIds: number[],
+  mediaEngine: MediaEngine,
+  signal: AbortSignal,
+): Promise<Map<number, ArrayBuffer>> => {
+  const audioRendition = mediaEngine.audioRendition;
+  if (!audioRendition) {
+    throw new Error("Audio rendition not available");
+  }
+
+  const segmentData = new Map<number, ArrayBuffer>();
+
+  // Fetch all segments - MediaEngine handles deduplication internally
+  const fetchPromises = segmentIds.map(async (segmentId) => {
+    const arrayBuffer = await mediaEngine.fetchMediaSegment(
+      segmentId,
+      audioRendition,
+      signal,
+    );
+    return [segmentId, arrayBuffer] as [number, ArrayBuffer];
+  });
+
+  const fetchedSegments = await Promise.all(fetchPromises);
+  signal.throwIfAborted();
+
+  for (const [segmentId, arrayBuffer] of fetchedSegments) {
+    segmentData.set(segmentId, arrayBuffer);
+  }
+
+  return segmentData;
+};
+
+/**
+ * Create audio span blob from init segment and media segments
+ * Pure function for blob creation
+ */
+const createAudioSpanBlob = (
+  initSegment: ArrayBuffer,
+  mediaSegments: ArrayBuffer[],
+): Blob => {
+  const chunks = [initSegment, ...mediaSegments];
+  return new Blob(chunks, { type: "audio/mp4" });
+};
+
+/**
+ * Fetch audio spanning a time range
+ * Main function that orchestrates segment calculation, fetching, and blob creation
+ */
+export const fetchAudioSpanningTime = async (
+  host: EFMedia,
+  fromMs: number,
+  toMs: number,
+  signal: AbortSignal,
+): Promise<AudioSpan> => {
+  // Validate inputs
+  if (fromMs >= toMs || fromMs < 0) {
+    throw new Error(`Invalid time range: fromMs=${fromMs}, toMs=${toMs}`);
+  }
+
+  // Get dependencies from host
+  const mediaEngine = await host.mediaEngineTask.taskComplete;
+  const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
+
+  if (!mediaEngine?.audioRendition) {
+    throw new Error("Audio rendition not available");
+  }
+
+  if (!initSegment) {
+    throw new Error("Audio init segment is not available");
+  }
+
+  // Calculate segments needed using the media engine's method
+  const segmentRanges = mediaEngine.calculateAudioSegmentRange(
+    fromMs,
+    toMs,
+    mediaEngine.audioRendition,
+    host.intrinsicDurationMs || 10000,
+  );
+
+  if (segmentRanges.length === 0) {
+    throw new Error(`No segments found for time range ${fromMs}-${toMs}ms`);
+  }
+
+  // Fetch segment data
+  const segmentIds = segmentRanges.map((r: SegmentTimeRange) => r.segmentId);
+  const segmentData = await fetchAudioSegmentData(
+    segmentIds,
+    mediaEngine,
+    signal,
+  );
+
+  // Create ordered array of segments
+  const orderedSegments = segmentIds.map((id: number) => {
+    const segment = segmentData.get(id);
+    if (!segment) {
+      throw new Error(`Missing segment data for segment ID ${id}`);
+    }
+    return segment;
+  });
+
+  // Create blob
+  const blob = createAudioSpanBlob(initSegment, orderedSegments);
+
+  // Calculate actual time boundaries
+  const actualStartMs = Math.min(
+    ...segmentRanges.map((r: SegmentTimeRange) => r.startMs),
+  );
+  const actualEndMs = Math.max(
+    ...segmentRanges.map((r: SegmentTimeRange) => r.endMs),
+  );
+
+  return {
+    startMs: actualStartMs,
+    endMs: actualEndMs,
+    blob,
+  };
+};