@remotion/whisper-webgpu 4.0.522 → 4.0.523

package/dist/esm/index.mjs

@@ -67,8 +67,6 @@ var canUseWhisperWebGpu = async () => {
   }
   return { supported: true };
 };
-// src/clear-stale-models.ts
-var clearStaleModels = () => Promise.resolve();
 // src/models.ts
 var WHISPER_WEBGPU_MODELS = [
   "tiny",
@@ -87,6 +85,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-tiny_timestamped",
     parameters: 39000000,
     multilingual: true,
+    supportsTranslation: true,
     webGpuDownloadSize: 119699015
   },
   "tiny.en": {
@@ -94,6 +93,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-tiny.en_timestamped",
     parameters: 39000000,
     multilingual: false,
+    supportsTranslation: false,
     webGpuDownloadSize: 119697479
   },
   base: {
@@ -101,6 +101,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-base_timestamped",
     parameters: 74000000,
     multilingual: true,
+    supportsTranslation: true,
     webGpuDownloadSize: 206190057
   },
   "base.en": {
@@ -108,6 +109,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-base.en_timestamped",
     parameters: 74000000,
     multilingual: false,
+    supportsTranslation: false,
     webGpuDownloadSize: 206188009
   },
   small: {
@@ -115,6 +117,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-small_timestamped",
     parameters: 244000000,
     multilingual: true,
+    supportsTranslation: true,
     webGpuDownloadSize: 586213010
   },
   "small.en": {
@@ -122,6 +125,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-small.en_timestamped",
     parameters: 244000000,
     multilingual: false,
+    supportsTranslation: false,
     webGpuDownloadSize: 586209938
   },
   medium: {
@@ -129,6 +133,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-medium_timestamped",
     parameters: 769000000,
     multilingual: true,
+    supportsTranslation: true,
     webGpuDownloadSize: 1698508143
   },
   "medium.en": {
@@ -136,6 +141,7 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-medium.en_timestamped",
     parameters: 769000000,
     multilingual: false,
+    supportsTranslation: false,
     webGpuDownloadSize: 1698504047
   },
   "large-v3-turbo": {
@@ -143,20 +149,121 @@ var MODEL_INFO = {
     modelId: "onnx-community/whisper-large-v3-turbo_timestamped",
     parameters: 809000000,
     multilingual: true,
-    webGpuDownloadSize: 2882584170
+    supportsTranslation: false,
+    webGpuDownloadSize: 1608611679
   }
 };
+var HOSTED_MODEL_IDS = {
+  tiny: "whisper-tiny_timestamped-v1",
+  "tiny.en": "whisper-tiny.en_timestamped-v1",
+  base: "whisper-base_timestamped-v1",
+  "base.en": "whisper-base.en_timestamped-v1",
+  small: "whisper-small_timestamped-v1",
+  "small.en": "whisper-small.en_timestamped-v1",
+  medium: "whisper-medium_timestamped-v1",
+  "medium.en": "whisper-medium.en_timestamped-v1",
+  "large-v3-turbo": "whisper-large-v3-turbo_timestamped-v1"
+};
 var getAvailableModels = () => {
   return WHISPER_WEBGPU_MODELS.map((model) => ({ ...MODEL_INFO[model] }));
 };
 var getModelInfo = (model) => {
   return MODEL_INFO[model];
 };
-var WHISPER_WEBGPU_DTYPE = {
+var getHostedModelId = (model) => {
+  return HOSTED_MODEL_IDS[model];
+};
+var DEFAULT_WHISPER_WEBGPU_DTYPE = {
   encoder_model: "fp32",
   decoder_model_merged: "q4"
 };
+var LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE = {
+  encoder_model: "fp16",
+  decoder_model_merged: "q4"
+};
+var getWhisperWebGpuDtype = (model) => {
+  return model === "large-v3-turbo" ? LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE : DEFAULT_WHISPER_WEBGPU_DTYPE;
+};
 
+// src/clear-stale-models.ts
+var legacyModelUrlPrefixes = getAvailableModels().map(({ modelId }) => `https://huggingface.co/${modelId}/resolve/main/`);
+var clearStaleModels = async () => {
+  if (typeof caches === "undefined") {
+    return;
+  }
+  const { env } = await import("@huggingface/transformers");
+  let cache;
+  let requests;
+  try {
+    cache = await caches.open(env.cacheKey);
+    requests = await cache.keys();
+  } catch {
+    return;
+  }
+  await Promise.all(requests.map((request) => {
+    if (legacyModelUrlPrefixes.some((prefix) => request.url.startsWith(prefix))) {
+      return cache.delete(request);
+    }
+    return Promise.resolve(false);
+  }));
+};
+// src/with-remotion-model-host.ts
+var REMOTION_MODEL_HOST = "https://remotion.media/";
+var REMOTION_MODEL_PATH_TEMPLATE = "models/{model}/";
+var REMOTION_MODEL_HOST_STATE = Symbol.for("@remotion/whisper-webgpu/model-host-state");
+var withRemotionModelHost = async (operation) => {
+  const transformers = await import("@huggingface/transformers");
+  const { env } = transformers;
+  const environmentWithState = env;
+  let state = environmentWithState[REMOTION_MODEL_HOST_STATE];
+  if (!state) {
+    state = {
+      activeOperations: 0,
+      previousRemoteConfiguration: null
+    };
+    Object.defineProperty(environmentWithState, REMOTION_MODEL_HOST_STATE, {
+      value: state
+    });
+  }
+  if (state.activeOperations === 0) {
+    state.previousRemoteConfiguration = {
+      remoteHost: env.remoteHost,
+      remotePathTemplate: env.remotePathTemplate
+    };
+    env.remoteHost = REMOTION_MODEL_HOST;
+    env.remotePathTemplate = REMOTION_MODEL_PATH_TEMPLATE;
+  }
+  state.activeOperations++;
+  try {
+    return await operation(transformers);
+  } finally {
+    state.activeOperations--;
+    if (state.activeOperations === 0) {
+      const configurationToRestore = state.previousRemoteConfiguration;
+      state.previousRemoteConfiguration = null;
+      if (configurationToRestore) {
+        if (env.remoteHost === REMOTION_MODEL_HOST) {
+          env.remoteHost = configurationToRestore.remoteHost;
+        }
+        if (env.remotePathTemplate === REMOTION_MODEL_PATH_TEMPLATE) {
+          env.remotePathTemplate = configurationToRestore.remotePathTemplate;
+        }
+      }
+    }
+  }
+};
+
+// src/is-whisper-model-cached.ts
+var isWhisperModelCached = ({
+  model
+}) => {
+  return withRemotionModelHost(({ ModelRegistry }) => {
+    return ModelRegistry.is_pipeline_cached("automatic-speech-recognition", getHostedModelId(model), {
+      device: "webgpu",
+      dtype: getWhisperWebGpuDtype(model)
+    });
+  });
+};
 // src/load-whisper-model.ts
 var pipelines = new Map;
 var getOrCreateWhisperPipeline = ({
@@ -167,81 +274,112 @@ var getOrCreateWhisperPipeline = ({
   const totalBytes = modelInfo.webGpuDownloadSize;
   const existing = pipelines.get(model);
   if (existing) {
-    return { state: existing, alreadyLoaded: true, totalBytes };
-  }
-  const loading = Promise.resolve().then(async () => {
-    const { pipeline } = await import("@huggingface/transformers");
-    onProgress?.({
-      status: "loading",
-      file: null,
-      progress: 0,
-      loadedBytes: 0,
-      totalBytes
-    });
-    const { modelId } = modelInfo;
-    const loadedByFile = new Map;
-    let lastProgress = 0;
-    let lastLoadedBytes = 0;
-    return pipeline("automatic-speech-recognition", modelId, {
-      device: "webgpu",
-      dtype: WHISPER_WEBGPU_DTYPE,
-      progress_callback: onProgress ? (event) => {
-        const record = event;
-        if (record.status === "progress" && typeof record.file === "string" && typeof record.loaded === "number" && Number.isFinite(record.loaded)) {
-          loadedByFile.set(record.file, Math.max(loadedByFile.get(record.file) ?? 0, record.loaded));
-          const loadedBytes = [...loadedByFile.values()].reduce((sum, loaded) => sum + loaded, 0);
-          lastProgress = Math.max(lastProgress, Math.min(loadedBytes / totalBytes, 0.99));
-          lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
-          onProgress({
-            status: "loading",
-            file: null,
-            progress: lastProgress,
-            loadedBytes: lastLoadedBytes,
-            totalBytes
-          });
-        }
-        if (record.status === "ready") {
-          onProgress({
-            status: "ready",
-            file: null,
-            progress: 1,
-            loadedBytes: Math.max(lastLoadedBytes, totalBytes),
-            totalBytes
-          });
+    if (onProgress) {
+      existing.progressListeners.add(onProgress);
+      if (existing.latestProgress) {
+        onProgress(existing.latestProgress);
+      }
+    }
+    return {
+      state: existing,
+      alreadyLoaded: true,
+      unsubscribe: () => {
+        if (onProgress) {
+          existing.progressListeners.delete(onProgress);
         }
-      } : undefined
-    });
-  });
+      }
+    };
+  }
+  const progressListeners = new Set;
+  if (onProgress) {
+    progressListeners.add(onProgress);
+  }
   const state = {
-    loading,
+    loading: Promise.resolve(null),
     activeTranscriptions: 0,
-    onIdle: []
+    onIdle: [],
+    progressListeners,
+    latestProgress: null
+  };
+  const emitProgress = (progress) => {
+    state.latestProgress = progress;
+    for (const listener of state.progressListeners) {
+      listener(progress);
+    }
   };
+  const loading = Promise.resolve().then(() => {
+    return withRemotionModelHost(({ pipeline }) => {
+      emitProgress({
+        status: "loading",
+        file: null,
+        progress: 0,
+        loadedBytes: 0,
+        totalBytes
+      });
+      const hostedModelId = getHostedModelId(model);
+      const loadedByFile = new Map;
+      let lastProgress = 0;
+      let lastLoadedBytes = 0;
+      return pipeline("automatic-speech-recognition", hostedModelId, {
+        device: "webgpu",
+        dtype: getWhisperWebGpuDtype(model),
+        progress_callback: (event) => {
+          const record = event;
+          if (record.status === "progress" && typeof record.file === "string" && typeof record.loaded === "number" && Number.isFinite(record.loaded)) {
+            loadedByFile.set(record.file, Math.max(loadedByFile.get(record.file) ?? 0, record.loaded));
+            const loadedBytes = [...loadedByFile.values()].reduce((sum, loaded) => sum + loaded, 0);
+            lastProgress = Math.max(lastProgress, Math.min(loadedBytes / totalBytes, 0.99));
+            lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
+            emitProgress({
+              status: "loading",
+              file: null,
+              progress: lastProgress,
+              loadedBytes: lastLoadedBytes,
+              totalBytes
+            });
+          }
+          if (record.status === "ready") {
+            emitProgress({
+              status: "ready",
+              file: null,
+              progress: 1,
+              loadedBytes: Math.max(lastLoadedBytes, totalBytes),
+              totalBytes
+            });
+          }
+        }
+      });
+    });
+  });
+  state.loading = loading;
   pipelines.set(model, state);
   loading.catch(() => {
     if (pipelines.get(model) === state) {
       pipelines.delete(model);
     }
   });
-  return { state, alreadyLoaded: false, totalBytes };
+  return {
+    state,
+    alreadyLoaded: false,
+    unsubscribe: () => {
+      if (onProgress) {
+        state.progressListeners.delete(onProgress);
+      }
+    }
+  };
 };
 var loadWhisperModel = async ({
   model,
   onProgress
 }) => {
-  const { state, alreadyLoaded, totalBytes } = getOrCreateWhisperPipeline({
+  const { state, alreadyLoaded, unsubscribe } = getOrCreateWhisperPipeline({
     model,
     onProgress
   });
-  await state.loading;
-  if (alreadyLoaded) {
-    onProgress?.({
-      status: "ready",
-      file: null,
-      progress: 1,
-      loadedBytes: totalBytes,
-      totalBytes
-    });
+  try {
+    await state.loading;
+  } finally {
+    unsubscribe();
   }
   return { alreadyLoaded };
 };
@@ -250,24 +388,17 @@ var withLoadedWhisperPipeline = async ({
   onProgress,
   run
 }) => {
-  const { state, alreadyLoaded, totalBytes } = getOrCreateWhisperPipeline({
+  const { state, unsubscribe } = getOrCreateWhisperPipeline({
     model,
     onProgress
   });
   state.activeTranscriptions++;
   try {
     const loaded = await state.loading;
-    if (alreadyLoaded) {
-      onProgress?.({
-        status: "ready",
-        file: null,
-        progress: 1,
-        loadedBytes: totalBytes,
-        totalBytes
-      });
-    }
+    unsubscribe();
     return await run(loaded);
   } finally {
+    unsubscribe();
     state.activeTranscriptions--;
     if (state.activeTranscriptions === 0) {
       for (const resolve of state.onIdle.splice(0)) {
@@ -297,15 +428,16 @@ var disposeWhisperModel = async ({
     await loadedPipeline.dispose();
   }));
 };
-// src/is-whisper-model-cached.ts
-var isWhisperModelCached = async ({
+// src/remove-whisper-model.ts
+var removeWhisperModel = async ({
   model
 }) => {
-  const { ModelRegistry } = await import("@huggingface/transformers");
-  const { modelId } = getModelInfo(model);
-  return ModelRegistry.is_pipeline_cached("automatic-speech-recognition", modelId, {
-    device: "webgpu",
-    dtype: WHISPER_WEBGPU_DTYPE
+  await disposeWhisperModel({ model });
+  await withRemotionModelHost(({ ModelRegistry }) => {
+    return ModelRegistry.clear_pipeline_cache("automatic-speech-recognition", getHostedModelId(model), {
+      device: "webgpu",
+      dtype: getWhisperWebGpuDtype(model)
+    });
   });
 };
 // src/resample-to-16khz.ts
@@ -356,8 +488,15 @@ var transcribe = async ({
   channelWaveform,
   model,
   language,
+  task = "transcribe",
   chunkLengthInSeconds = 30,
   strideLengthInSeconds = 5,
+  forceFullSequences = false,
+  doSample = false,
+  temperature = 1,
+  topK = 50,
+  repetitionPenalty = 1,
+  noRepeatNgramSize = 0,
   onModelLoadProgress
 }) => {
   if (channelWaveform.length === 0) {
@@ -369,7 +508,31 @@ var transcribe = async ({
   if (!Number.isFinite(strideLengthInSeconds) || strideLengthInSeconds < 0 || strideLengthInSeconds * 2 >= chunkLengthInSeconds) {
     throw new Error("strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.");
   }
-  const { multilingual } = getModelInfo(model);
+  if (typeof forceFullSequences !== "boolean") {
+    throw new TypeError("forceFullSequences must be a boolean.");
+  }
+  if (typeof doSample !== "boolean") {
+    throw new TypeError("doSample must be a boolean.");
+  }
+  if (!Number.isFinite(temperature) || temperature <= 0) {
+    throw new TypeError("temperature must be a finite number greater than 0.");
+  }
+  if (!Number.isInteger(topK) || topK < 0) {
+    throw new TypeError("topK must be a non-negative integer.");
+  }
+  if (!Number.isFinite(repetitionPenalty) || repetitionPenalty <= 0) {
+    throw new TypeError("repetitionPenalty must be a finite number greater than 0.");
+  }
+  if (!Number.isInteger(noRepeatNgramSize) || noRepeatNgramSize < 0) {
+    throw new TypeError("noRepeatNgramSize must be a non-negative integer.");
+  }
+  if (task !== "transcribe" && task !== "translate") {
+    throw new TypeError('task must be either "transcribe" or "translate".');
+  }
+  const { multilingual, supportsTranslation } = getModelInfo(model);
+  if (task === "translate" && !supportsTranslation) {
+    throw new Error(`The model "${model}" does not support translation.`);
+  }
   if (multilingual && (language === undefined || language === "auto")) {
     throw new Error(`The language option is required for the multilingual model "${model}" because automatic language detection is not supported.`);
   }
@@ -384,7 +547,13 @@ var transcribe = async ({
         return_timestamps: "word",
         chunk_length_s: chunkLengthInSeconds,
         stride_length_s: strideLengthInSeconds,
-        ...multilingual ? { language } : {}
+        force_full_sequences: forceFullSequences,
+        do_sample: doSample,
+        temperature,
+        top_k: topK,
+        repetition_penalty: repetitionPenalty,
+        no_repeat_ngram_size: noRepeatNgramSize,
+        ...multilingual ? { language, task } : {}
       });
     }
   });
@@ -421,6 +590,7 @@ export {
   transcribe,
   toCaptions,
   resampleTo16Khz,
+  removeWhisperModel,
   loadWhisperModel,
   isWhisperModelCached,
   getAvailableModels,