@remotion/studio 4.0.521 → 4.0.523

package/dist/esm/CaptionQueueProcessor-k0yrm3dj.mjs

@@ -0,0 +1,211 @@
+import {
+  loadModelForJob
+} from "./index-v4tr1bft.mjs";
+import {
+  RenderQueueContext
+} from "./index-9x6e1dq0.mjs";
+import {
+  writeStaticFile
+} from "./index-cw0pnpg5.mjs";
+import"./index-rcv7qkt5.mjs";
+
+// src/components/RenderQueue/CaptionQueueProcessor.tsx
+import {
+  canUseWhisperWebGpu,
+  clearStaleModels,
+  disposeWhisperModel,
+  isWhisperModelCached,
+  loadWhisperModel,
+  toCaptions,
+  transcribe
+} from "@remotion/whisper-webgpu";
+import { useCallback, useContext, useEffect } from "react";
+
+// src/components/Transcription/resample-media-to-16-khz.ts
+import { WHISPER_WEBGPU_SAMPLE_RATE } from "@remotion/whisper-webgpu";
+import {
+  ALL_FORMATS,
+  Conversion,
+  Input,
+  NullTarget,
+  Output,
+  UrlSource,
+  WavOutputFormat
+} from "mediabunny";
+var resampleMediaTo16Khz = async ({
+  src,
+  audioStreamIndex,
+  requestInit,
+  onProgress
+}) => {
+  const input = new Input({
+    formats: ALL_FORMATS,
+    source: new UrlSource(src, requestInit === null ? {} : { requestInit })
+  });
+  const waveformChunks = [];
+  try {
+    onProgress(0);
+    const audioTracks = await input.getAudioTracks();
+    const primaryVideoTrack = audioStreamIndex === null ? await input.getPrimaryVideoTrack() : null;
+    const audioTrack = audioStreamIndex !== null ? audioTracks[audioStreamIndex] ?? null : primaryVideoTrack ? await primaryVideoTrack.getPrimaryPairableAudioTrack() : audioTracks[0] ?? null;
+    if (audioTrack === null) {
+      throw new Error("The selected media does not have an audio track.");
+    }
+    if (await audioTrack.isLive()) {
+      throw new Error("Live media cannot be transcribed.");
+    }
+    const output = new Output({
+      format: new WavOutputFormat,
+      target: new NullTarget
+    });
+    const conversion = await Conversion.init({
+      input,
+      output,
+      tracks: "all",
+      video: { discard: true },
+      audio: (track) => {
+        if (track.id !== audioTrack.id) {
+          return { discard: true };
+        }
+        return {
+          codec: "pcm-f32",
+          forceTranscode: true,
+          numberOfChannels: 1,
+          process: (sample) => {
+            const floats = new Float32Array(sample.allocationSize({ format: "f32", planeIndex: 0 }) / Float32Array.BYTES_PER_ELEMENT);
+            sample.copyTo(floats, { format: "f32", planeIndex: 0 });
+            waveformChunks.push({
+              startFrame: Math.round(sample.timestamp * WHISPER_WEBGPU_SAMPLE_RATE),
+              waveform: floats
+            });
+            return sample;
+          },
+          sampleFormat: "f32",
+          sampleRate: WHISPER_WEBGPU_SAMPLE_RATE
+        };
+      }
+    });
+    if (!conversion.isValid) {
+      throw new Error("The audio track cannot be decoded in this browser. Try converting the media to WAV first.");
+    }
+    conversion.onProgress = (progress) => onProgress(progress);
+    await conversion.execute();
+    onProgress(1);
+  } finally {
+    input.dispose();
+  }
+  const waveformLength = waveformChunks.reduce((maximum, chunk) => Math.max(maximum, chunk.startFrame + chunk.waveform.length), 0);
+  const waveform = new Float32Array(waveformLength);
+  for (const chunk of waveformChunks) {
+    const destinationStart = Math.max(0, chunk.startFrame);
+    const sourceStart = Math.max(0, -chunk.startFrame);
+    const availableLength = Math.min(chunk.waveform.length - sourceStart, waveform.length - destinationStart);
+    if (availableLength > 0) {
+      waveform.set(chunk.waveform.subarray(sourceStart, sourceStart + availableLength), destinationStart);
+    }
+  }
+  return waveform;
+};
+
+// src/components/RenderQueue/CaptionQueueProcessor.tsx
+var CaptionQueueProcessor = () => {
+  const {
+    updateCaptionJobProgress,
+    markCaptionJobDone,
+    markCaptionJobFailed,
+    setProcessCaptionJobCallback
+  } = useContext(RenderQueueContext);
+  const processJob = useCallback(async (job) => {
+    let captionCount = null;
+    let processingError = null;
+    try {
+      updateCaptionJobProgress(job.id, {
+        message: "Checking WebGPU support...",
+        value: 0
+      });
+      const support = await canUseWhisperWebGpu();
+      if (!support.supported) {
+        throw new Error(support.detailedReason);
+      }
+      updateCaptionJobProgress(job.id, {
+        message: `Checking ${job.model}...`,
+        value: 0.02
+      });
+      await clearStaleModels();
+      await loadModelForJob({
+        model: job.model,
+        progressStart: 0.03,
+        progressSpan: 0.27,
+        isModelCached: (model) => isWhisperModelCached({ model }),
+        loadModel: (model, onProgress) => loadWhisperModel({
+          model,
+          onProgress: (progress) => onProgress(progress.progress)
+        }),
+        updateProgress: (progress) => updateCaptionJobProgress(job.id, progress)
+      });
+      updateCaptionJobProgress(job.id, {
+        message: `Preparing ${job.displayName}...`,
+        value: 0.3
+      });
+      const channelWaveform = await resampleMediaTo16Khz({
+        src: job.src,
+        audioStreamIndex: job.audioStreamIndex,
+        requestInit: job.requestInit,
+        onProgress: (progress) => {
+          updateCaptionJobProgress(job.id, {
+            message: `Preparing ${job.displayName}...`,
+            value: 0.3 + progress * 0.2
+          });
+        }
+      });
+      updateCaptionJobProgress(job.id, {
+        message: `Transcribing ${job.displayName}...`,
+        value: 0.5
+      });
+      const transcription = await transcribe({
+        channelWaveform,
+        model: job.model,
+        task: job.task,
+        chunkLengthInSeconds: job.chunkLengthInSeconds,
+        strideLengthInSeconds: job.strideLengthInSeconds,
+        forceFullSequences: job.forceFullSequences,
+        doSample: job.doSample,
+        temperature: job.temperature,
+        topK: job.topK,
+        repetitionPenalty: job.repetitionPenalty,
+        noRepeatNgramSize: job.noRepeatNgramSize,
+        ...job.language === null ? {} : { language: job.language }
+      });
+      const { captions } = toCaptions({
+        whisperWebGpuOutput: transcription
+      });
+      updateCaptionJobProgress(job.id, {
+        message: `Saving ${job.outName}...`,
+        value: 0.95
+      });
+      await writeStaticFile({
+        contents: JSON.stringify(captions, null, 2),
+        filePath: job.outName
+      });
+      captionCount = captions.length;
+    } catch (error) {
+      processingError = error instanceof Error ? error : new Error(String(error));
+    }
+    try {
+      await disposeWhisperModel({ model: job.model });
+    } catch {}
+    if (processingError) {
+      markCaptionJobFailed(job.id, processingError);
+    } else {
+      markCaptionJobDone(job.id, captionCount ?? 0);
+    }
+  }, [markCaptionJobDone, markCaptionJobFailed, updateCaptionJobProgress]);
+  useEffect(() => {
+    setProcessCaptionJobCallback(processJob);
+    return () => setProcessCaptionJobCallback(null);
+  }, [processJob, setProcessCaptionJobCallback]);
+  return null;
+};
+export {
+  CaptionQueueProcessor
+};