@editframe/elements 0.26.2-beta.0 → 0.26.4-beta.0

package/src/elements/EFMedia/audioTasks/makeAudioBufferTask.ts

@@ -1,117 +0,0 @@
-import { Task } from "@lit/task";
-
-import { EF_INTERACTIVE } from "../../../EF_INTERACTIVE";
-import { EF_RENDERING } from "../../../EF_RENDERING";
-import type { AudioRendition } from "../../../transcoding/types";
-import type { EFMedia } from "../../EFMedia";
-import {
-  type MediaBufferConfig,
-  type MediaBufferState,
-  manageMediaBuffer,
-} from "../shared/BufferUtils";
-import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
-
-/**
- * Configuration for audio buffering - extends the generic interface
- */
-export interface AudioBufferConfig extends MediaBufferConfig {}
-
-/**
- * State of the audio buffer - uses the generic interface
- */
-export interface AudioBufferState extends MediaBufferState {}
-
-type AudioBufferTask = Task<readonly [number], AudioBufferState>;
-export const makeAudioBufferTask = (host: EFMedia): AudioBufferTask => {
-  let currentState: AudioBufferState = {
-    currentSeekTimeMs: 0,
-    requestedSegments: new Set(),
-    activeRequests: new Set(),
-    requestQueue: [],
-  };
-
-  return new Task(host, {
-    autoRun: EF_INTERACTIVE, // Make lazy - only run when element becomes timeline-active
-    args: () => [host.desiredSeekTimeMs] as const,
-    onError: (error) => {
-      console.error("audioBufferTask error", error);
-    },
-    onComplete: (value) => {
-      currentState = value;
-    },
-    task: async ([seekTimeMs], { signal }) => {
-      // Skip buffering entirely in rendering mode
-      if (EF_RENDERING()) {
-        return currentState; // Return existing state without any buffering activity
-      }
-
-      // Get media engine to potentially override buffer configuration
-      const mediaEngine = await getLatestMediaEngine(host, signal);
-
-      // Return existing state if no audio rendition available
-      if (!mediaEngine.audioRendition) {
-        return currentState;
-      }
-
-      // Use media engine's buffer config, falling back to host properties
-      const engineConfig = mediaEngine.getBufferConfig();
-      const bufferDurationMs = engineConfig.audioBufferDurationMs;
-      const maxParallelFetches = engineConfig.maxAudioBufferFetches;
-
-      const currentConfig: AudioBufferConfig = {
-        bufferDurationMs,
-        maxParallelFetches,
-        enableBuffering: host.enableAudioBuffering,
-        bufferThresholdMs: engineConfig.bufferThresholdMs,
-      };
-
-      // Timeline context for priority-based buffering
-      const timelineContext =
-        host.rootTimegroup?.currentTimeMs !== undefined
-          ? {
-              elementStartMs: host.startTimeMs,
-              elementEndMs: host.endTimeMs,
-              playheadMs: host.rootTimegroup.currentTimeMs,
-            }
-          : undefined;
-
-      return manageMediaBuffer<AudioRendition>(
-        seekTimeMs,
-        currentConfig,
-        currentState,
-        (host as any).intrinsicDurationMs || 10000,
-        signal,
-        {
-          computeSegmentId: async (timeMs, rendition) => {
-            // Use media engine's computeSegmentId
-            const mediaEngine = await getLatestMediaEngine(host, signal);
-            return mediaEngine.computeSegmentId(timeMs, rendition);
-          },
-          prefetchSegment: async (segmentId, rendition) => {
-            // Trigger prefetch through BaseMediaEngine - let it handle caching
-            const mediaEngine = await getLatestMediaEngine(host, signal);
-            await mediaEngine.fetchMediaSegment(segmentId, rendition);
-            // Don't return data - just ensure it's cached in BaseMediaEngine
-          },
-          isSegmentCached: (segmentId, rendition) => {
-            // Check if segment is already cached in BaseMediaEngine
-            const mediaEngine = host.mediaEngineTask.value;
-            if (!mediaEngine) return false;
-
-            return mediaEngine.isSegmentCached(segmentId, rendition);
-          },
-          getRendition: async () => {
-            const mediaEngine = await getLatestMediaEngine(host, signal);
-            const audioRendition = mediaEngine.audioRendition;
-            if (!audioRendition) {
-              throw new Error("Audio rendition not available");
-            }
-            return audioRendition;
-          },
-          logError: console.error,
-        },
-        timelineContext,
-      );
-    },
-  });
-};

package/src/elements/EFMedia/audioTasks/makeAudioFrequencyAnalysisTask.ts

@@ -1,246 +0,0 @@
-import { Task } from "@lit/task";
-import { EF_INTERACTIVE } from "../../../EF_INTERACTIVE.js";
-import { LRUCache } from "../../../utils/LRUCache.js";
-import type { EFMedia } from "../../EFMedia.js";
-
-// DECAY_WEIGHT constant - same as original
-const DECAY_WEIGHT = 0.8;
-
-function processFFTData(
-  fftData: Uint8Array,
-  zeroThresholdPercent = 0.1,
-): Uint8Array {
-  // Step 1: Determine the threshold for zeros
-  const totalBins = fftData.length;
-  const zeroThresholdCount = Math.floor(totalBins * zeroThresholdPercent);
-
-  // Step 2: Interrogate the FFT output to find the cutoff point
-  let zeroCount = 0;
-  let cutoffIndex = totalBins; // Default to the end of the array
-
-  for (let i = totalBins - 1; i >= 0; i--) {
-    if (fftData[i] ?? 0 < 10) {
-      zeroCount++;
-    } else {
-      // If we encounter a non-zero value, we can stop
-      if (zeroCount >= zeroThresholdCount) {
-        cutoffIndex = i + 1; // Include this index
-        break;
-      }
-    }
-  }
-
-  if (cutoffIndex < zeroThresholdCount) {
-    return fftData;
-  }
-
-  // Step 3: Resample the "good" portion of the data
-  const goodData = fftData.slice(0, cutoffIndex);
-  const resampledData = interpolateData(goodData, fftData.length);
-
-  // Step 4: Attenuate the top 10% of interpolated samples
-  const attenuationStartIndex = Math.floor(totalBins * 0.9);
-  for (let i = attenuationStartIndex; i < totalBins; i++) {
-    // Calculate attenuation factor that goes from 1 to 0 over the top 10%
-    const attenuationProgress =
-      (i - attenuationStartIndex) / (totalBins - attenuationStartIndex) + 0.2;
-    const attenuationFactor = Math.max(0, 1 - attenuationProgress);
-    resampledData[i] = Math.floor((resampledData[i] ?? 0) * attenuationFactor);
-  }
-
-  return resampledData;
-}
-
-function interpolateData(data: Uint8Array, targetSize: number): Uint8Array {
-  const resampled = new Uint8Array(targetSize);
-  const dataLength = data.length;
-
-  for (let i = 0; i < targetSize; i++) {
-    // Calculate the corresponding index in the original data
-    const ratio = (i / (targetSize - 1)) * (dataLength - 1);
-    const index = Math.floor(ratio);
-    const fraction = ratio - index;
-
-    // Handle edge cases
-    if (index >= dataLength - 1) {
-      resampled[i] = data[dataLength - 1] ?? 0; // Last value
-    } else {
-      // Linear interpolation
-      resampled[i] = Math.round(
-        (data[index] ?? 0) * (1 - fraction) + (data[index + 1] ?? 0) * fraction,
-      );
-    }
-  }
-
-  return resampled;
-}
-
-export function makeAudioFrequencyAnalysisTask(element: EFMedia) {
-  // Internal cache for this task instance (same as original #frequencyDataCache)
-  const cache = new LRUCache<string, Uint8Array>(100);
-
-  return new Task(element, {
-    autoRun: EF_INTERACTIVE,
-    onError: (error) => {
-      console.error("frequencyDataTask error", error);
-    },
-    args: () =>
-      [
-        element.currentSourceTimeMs,
-        element.fftSize,
-        element.fftDecay,
-        element.fftGain,
-        element.shouldInterpolateFrequencies,
-      ] as const,
-    task: async (_, { signal }) => {
-      if (element.currentSourceTimeMs < 0) return null;
-
-      const currentTimeMs = element.currentSourceTimeMs;
-
-      // Calculate exact audio window needed based on fftDecay and frame timing
-      const frameIntervalMs = 1000 / 30; // 33.33ms per frame
-
-      // Need audio from earliest frame to current frame
-      const earliestFrameMs =
-        currentTimeMs - (element.fftDecay - 1) * frameIntervalMs;
-      const fromMs = Math.max(0, earliestFrameMs);
-      const maxToMs = currentTimeMs + frameIntervalMs; // Include current frame
-      const videoDurationMs = element.intrinsicDurationMs || 0;
-      const toMs =
-        videoDurationMs > 0 ? Math.min(maxToMs, videoDurationMs) : maxToMs;
-
-      // If the clamping results in an invalid range (seeking beyond the end), skip analysis silently
-      if (fromMs >= toMs) {
-        return null;
-      }
-
-      // Check cache early - before expensive audio fetching
-      // Use a preliminary cache key that doesn't depend on actual startOffsetMs from audio span
-      const preliminaryCacheKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftDecay}:${element.fftGain}:${fromMs}:${currentTimeMs}`;
-      const cachedSmoothedData = cache.get(preliminaryCacheKey);
-      if (cachedSmoothedData) {
-        return cachedSmoothedData;
-      }
-
-      const { fetchAudioSpanningTime: fetchAudioSpan } = await import(
-        "../shared/AudioSpanUtils.ts"
-      );
-      const audioSpan = await fetchAudioSpan(element, fromMs, toMs, signal);
-
-      if (!audioSpan || !audioSpan.blob) {
-        console.warn("Frequency analysis skipped: no audio data available");
-        return null;
-      }
-
-      // Decode the real audio data
-      const tempAudioContext = new OfflineAudioContext(2, 48000, 48000);
-      const arrayBuffer = await audioSpan.blob.arrayBuffer();
-      const audioBuffer = await tempAudioContext.decodeAudioData(arrayBuffer);
-
-      // Use actual startOffset from audioSpan (relative to requested time)
-      const startOffsetMs = audioSpan.startMs;
-
-      const framesData = await Promise.all(
-        Array.from({ length: element.fftDecay }, async (_, i) => {
-          const frameOffset = i * (1000 / 30);
-          const startTime = Math.max(
-            0,
-            (currentTimeMs - frameOffset - startOffsetMs) / 1000,
-          );
-
-          // Cache key for this specific frame
-          const cacheKey = `${element.shouldInterpolateFrequencies}:${element.fftSize}:${element.fftGain}:${startOffsetMs}:${startTime}`;
-
-          // Check cache for this specific frame
-          const cachedFrame = cache.get(cacheKey);
-          if (cachedFrame) {
-            return cachedFrame;
-          }
-
-          // Running 48000 * (1 / 30) = 1600 broke something terrible, it came out as 0,
-          // I'm assuming weird floating point nonsense to do with running on rosetta
-          const SIZE = 48000 / 30;
-          let audioContext: OfflineAudioContext;
-          try {
-            audioContext = new OfflineAudioContext(2, SIZE, 48000);
-          } catch (error) {
-            throw new Error(
-              `[EFMedia.frequencyDataTask] Failed to create OfflineAudioContext(2, ${SIZE}, 48000) for frame ${i} at time ${startTime}s: ${error instanceof Error ? error.message : String(error)}. This is for audio frequency analysis.`,
-            );
-          }
-          const analyser = audioContext.createAnalyser();
-          analyser.fftSize = element.fftSize;
-          analyser.minDecibels = -90;
-          analyser.maxDecibels = -10;
-
-          const gainNode = audioContext.createGain();
-          gainNode.gain.value = element.fftGain;
-
-          const filter = audioContext.createBiquadFilter();
-          filter.type = "bandpass";
-          filter.frequency.value = 15000;
-          filter.Q.value = 0.05;
-
-          const audioBufferSource = audioContext.createBufferSource();
-          audioBufferSource.buffer = audioBuffer;
-
-          audioBufferSource.connect(filter);
-          filter.connect(gainNode);
-          gainNode.connect(analyser);
-          analyser.connect(audioContext.destination);
-
-          audioBufferSource.start(0, startTime, 1 / 30);
-
-          try {
-            await audioContext.startRendering();
-            const frameData = new Uint8Array(element.fftSize / 2);
-            analyser.getByteFrequencyData(frameData);
-
-            // Cache this frame's analysis
-            cache.set(cacheKey, frameData);
-            return frameData;
-          } finally {
-            audioBufferSource.disconnect();
-            analyser.disconnect();
-          }
-        }),
-      );
-
-      const frameLength = framesData[0]?.length ?? 0;
-
-      // Combine frames with decay
-      const smoothedData = new Uint8Array(frameLength);
-      for (let i = 0; i < frameLength; i++) {
-        let weightedSum = 0;
-        let weightSum = 0;
-
-        framesData.forEach((frame: Uint8Array, frameIndex: number) => {
-          const decayWeight = DECAY_WEIGHT ** frameIndex;
-          weightedSum += (frame[i] ?? 0) * decayWeight;
-          weightSum += decayWeight;
-        });
-
-        smoothedData[i] = Math.min(255, Math.round(weightedSum / weightSum));
-      }
-
-      // Apply frequency weights using instance FREQ_WEIGHTS
-      smoothedData.forEach((value, i) => {
-        const freqWeight = element.FREQ_WEIGHTS[i] ?? 0;
-        smoothedData[i] = Math.min(255, Math.round(value * freqWeight));
-      });
-
-      // Only return the lower half of the frequency data
-      // The top half is zeroed out, which makes for aesthetically unpleasing waveforms
-      const slicedData = smoothedData.slice(
-        0,
-        Math.floor(smoothedData.length / 2),
-      );
-      const processedData = element.shouldInterpolateFrequencies
-        ? processFFTData(slicedData)
-        : slicedData;
-      // Cache with the preliminary key so future requests can skip audio fetching
-      cache.set(preliminaryCacheKey, processedData);
-      return processedData;
-    },
-  });
-}

package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.browsertest.ts

@@ -1,59 +0,0 @@
-import { TaskStatus } from "@lit/task";
-import { customElement } from "lit/decorators.js";
-import { afterEach, beforeEach, describe, vi } from "vitest";
-import { test as baseTest } from "../../../../test/useMSW.js";
-import { EFAudio } from "../../EFAudio";
-import { makeAudioInitSegmentFetchTask } from "./makeAudioInitSegmentFetchTask";
-
-@customElement("test-media-audio-init-segment-fetch")
-class TestMediaAudioInitSegmentFetch extends EFAudio {}
-
-declare global {
-  interface HTMLElementTagNameMap {
-    "test-media-audio-init-segment-fetch": TestMediaAudioInitSegmentFetch;
-  }
-}
-
-const test = baseTest.extend<{
-  element: TestMediaAudioInitSegmentFetch;
-}>({
-  element: async ({}, use) => {
-    const element = document.createElement(
-      "test-media-audio-init-segment-fetch",
-    );
-    await use(element);
-    element.remove();
-  },
-});
-
-describe("makeAudioInitSegmentFetchTask", () => {
-  beforeEach(() => {
-    // MSW setup is now handled by test fixtures
-  });
-
-  afterEach(() => {
-    const elements = document.querySelectorAll(
-      "test-media-audio-init-segment-fetch",
-    );
-    for (const element of elements) {
-      element.remove();
-    }
-    vi.restoreAllMocks();
-  });
-
-  test("creates task with correct initial state", ({ element, expect }) => {
-    const task = makeAudioInitSegmentFetchTask(element);
-
-    expect(task).toBeDefined();
-    expect(task.status).toBe(TaskStatus.INITIAL);
-    expect(task.value).toBeUndefined();
-    expect(task.error).toBeUndefined();
-  });
-
-  test("task integrates with element properties", ({ element, expect }) => {
-    const task = makeAudioInitSegmentFetchTask(element);
-
-    expect(task).toBeDefined();
-    expect(task.status).toBe(TaskStatus.INITIAL);
-  });
-});

package/src/elements/EFMedia/audioTasks/makeAudioInitSegmentFetchTask.ts

@@ -1,27 +0,0 @@
-import { Task } from "@lit/task";
-import type { MediaEngine } from "../../../transcoding/types";
-import type { EFMedia } from "../../EFMedia";
-import { getLatestMediaEngine } from "../tasks/makeMediaEngineTask";
-
-export const makeAudioInitSegmentFetchTask = (
-  host: EFMedia,
-): Task<readonly [MediaEngine | undefined], ArrayBuffer | undefined> => {
-  return new Task(host, {
-    args: () => [host.mediaEngineTask.value] as const,
-    onError: (error) => {
-      console.error("audioInitSegmentFetchTask error", error);
-    },
-    onComplete: (_value) => {},
-    task: async ([_mediaEngine], { signal }) => {
-      const mediaEngine = await getLatestMediaEngine(host, signal);
-      const audioRendition = mediaEngine.getAudioRendition();
-
-      // Return undefined if no audio rendition available (video-only asset)
-      if (!audioRendition) {
-        return undefined;
-      }
-
-      return mediaEngine.fetchInitSegment(audioRendition, signal);
-    },
-  });
-};

package/src/elements/EFMedia/audioTasks/makeAudioInputTask.browsertest.ts

@@ -1,55 +0,0 @@
-import { TaskStatus } from "@lit/task";
-import { customElement } from "lit/decorators.js";
-import { afterEach, beforeEach, describe, vi } from "vitest";
-import { test as baseTest } from "../../../../test/useMSW.js";
-import { EFAudio } from "../../EFAudio";
-import { makeAudioInputTask } from "./makeAudioInputTask";
-
-@customElement("test-media-audio-input")
-class TestMediaAudioInput extends EFAudio {}
-
-declare global {
-  interface HTMLElementTagNameMap {
-    "test-media-audio-input": TestMediaAudioInput;
-  }
-}
-
-const test = baseTest.extend<{
-  element: TestMediaAudioInput;
-}>({
-  element: async ({}, use) => {
-    const element = document.createElement("test-media-audio-input");
-    await use(element);
-    element.remove();
-  },
-});
-
-describe("makeAudioInputTask", () => {
-  beforeEach(() => {
-    // MSW setup is now handled by test fixtures
-  });
-
-  afterEach(() => {
-    const elements = document.querySelectorAll("test-media-audio-input");
-    for (const element of elements) {
-      element.remove();
-    }
-    vi.restoreAllMocks();
-  });
-
-  test("creates task with correct initial state", ({ element, expect }) => {
-    const task = makeAudioInputTask(element);
-
-    expect(task).toBeDefined();
-    expect(task.status).toBe(TaskStatus.INITIAL);
-    expect(task.value).toBeUndefined();
-    expect(task.error).toBeUndefined();
-  });
-
-  test("task integrates with element properties", ({ element, expect }) => {
-    const task = makeAudioInputTask(element);
-
-    expect(task).toBeDefined();
-    expect(task.status).toBe(TaskStatus.INITIAL);
-  });
-});

package/src/elements/EFMedia/audioTasks/makeAudioInputTask.ts

@@ -1,53 +0,0 @@
-import { Task } from "@lit/task";
-import { EFMedia } from "../../EFMedia";
-import { BufferedSeekingInput } from "../BufferedSeekingInput";
-import type { InputTask } from "../shared/MediaTaskUtils";
-
-export const makeAudioInputTask = (host: EFMedia): InputTask => {
-  return new Task<
-    readonly [ArrayBuffer | undefined, ArrayBuffer | undefined],
-    BufferedSeekingInput | undefined
-  >(host, {
-    args: () =>
-      [
-        host.audioInitSegmentFetchTask.value,
-        host.audioSegmentFetchTask.value,
-      ] as const,
-    onError: (error) => {
-      console.error("audioInputTask error", error);
-    },
-    onComplete: (_value) => {},
-    task: async (_, { signal }) => {
-      const mediaEngine = await host.mediaEngineTask.taskComplete;
-      if (signal.aborted) return undefined;
-
-      const audioRendition = mediaEngine?.audioRendition;
-
-      // Return undefined if no audio rendition available (video-only asset)
-      if (!audioRendition) {
-        return undefined;
-      }
-
-      const initSegment = await host.audioInitSegmentFetchTask.taskComplete;
-      if (signal.aborted) return undefined;
-
-      const segment = await host.audioSegmentFetchTask.taskComplete;
-      if (signal.aborted) return undefined;
-
-      if (!initSegment || !segment) {
-        return undefined;
-      }
-
-      const startTimeOffsetMs = audioRendition.startTimeOffsetMs;
-
-      const arrayBuffer = await new Blob([initSegment, segment]).arrayBuffer();
-      if (signal.aborted) return undefined;
-
-      return new BufferedSeekingInput(arrayBuffer, {
-        videoBufferSize: EFMedia.VIDEO_SAMPLE_BUFFER_SIZE,
-        audioBufferSize: EFMedia.AUDIO_SAMPLE_BUFFER_SIZE,
-        startTimeOffsetMs,
-      });
-    },
-  });
-};