@remotion/whisper-web 4.0.302

package/emscripten.cpp

@@ -0,0 +1,303 @@
+#include "whisper.h"
+
+#include <emscripten.h>
+#include <emscripten/bind.h>
+#include <iostream>
+#include <vector>
+#include <thread>
+
+std::thread g_worker;
+
+static inline int mpow2(int n) {
+    int p = 1;
+    while (p <= n) p *= 2;
+    return p / 2;
+}
+
+static char * escape_double_quotes_and_backslashes(const char * str) {
+    if (str == NULL) {
+        return NULL;
+    }
+
+    size_t escaped_length = strlen(str) + 1;
+
+    for (size_t i = 0; str[i] != '\0'; i++) {
+        if (str[i] == '"' || str[i] == '\\') {
+            escaped_length++;
+        }
+    }
+
+    char * escaped = (char *)calloc(escaped_length, 1); // pre-zeroed
+    if (escaped == NULL) {
+        return NULL;
+    }
+
+    size_t pos = 0;
+    for (size_t i = 0; str[i] != '\0'; i++) {
+        if (str[i] == '"' || str[i] == '\\') {
+            escaped[pos++] = '\\';
+        }
+        escaped[pos++] = str[i];
+    }
+
+    // no need to set zero due to calloc() being used prior
+
+    return escaped;
+}
+
+//  500 -> 00:05.000
+// 6000 -> 01:00.000
+std::string to_timestamp(int64_t t, bool comma) {
+    int64_t msec = t * 10;
+    int64_t hr = msec / (1000 * 60 * 60);
+    msec = msec - hr * (1000 * 60 * 60);
+    int64_t min = msec / (1000 * 60);
+    msec = msec - min * (1000 * 60);
+    int64_t sec = msec / 1000;
+    msec = msec - sec * 1000;
+
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%02d:%02d:%02d%s%03d", (int) hr, (int) min, (int) sec, comma ? "," : ".", (int) msec);
+
+    return std::string(buf);
+}
+
+static bool output_json(
+             struct whisper_context * ctx,
+                               bool   final,
+                               int    first_segment,
+                               int    n_segments) {
+    int indent = 0;
+    std::string output;
+
+
+    auto doindent = [&]() {
+        for (int i = 0; i < indent; i++) output += "\t";
+    };
+
+    auto start_arr = [&](const char *name) {
+        doindent();
+        output += "\"" + std::string(name) + "\": [";
+        indent++;
+    };
+
+    auto end_arr = [&](bool end) {
+        indent--;
+        doindent();
+        output += (end ? "]" : "],");
+    };
+
+    auto start_obj = [&](const char *name) {
+        doindent();
+        if (name) {
+            output += "\"" + std::string(name) + "\": {";
+        } else {
+            output += "{";
+        }
+        indent++;
+    };
+
+    auto end_obj = [&](bool end) {
+        indent--;
+        doindent();
+        output += (end ? "}" : "},");
+    };
+
+    auto start_value = [&](const char *name) {
+        doindent();
+        output += "\"" + std::string(name) + "\": ";
+    };
+
+    auto value_s = [&](const char *name, const char *val, bool end) {
+        start_value(name);
+        char * val_escaped = escape_double_quotes_and_backslashes(val);
+        output += "\"" + std::string(val_escaped) + (end ? "\"" : "\",");
+        free(val_escaped);
+    };
+
+    auto end_value = [&](bool end) {
+        output += (end ? "" : ",");
+    };
+
+    auto value_i = [&](const char *name, const int64_t val, bool end) {
+        start_value(name);
+        output += std::to_string(val);
+        end_value(end);
+    };
+
+    auto value_f = [&](const char *name, const float val, bool end) {
+        start_value(name);
+        output += std::to_string(val);
+        end_value(end);
+    };
+
+    auto value_b = [&](const char *name, const bool val, bool end) {
+        start_value(name);
+        output += (val ? "true" : "false");
+        end_value(end);
+    };
+
+    auto times_o = [&](int64_t t0, int64_t t1, bool end) {
+        start_obj("timestamps");
+        value_s("from", to_timestamp(t0, true).c_str(), false);
+        value_s("to", to_timestamp(t1, true).c_str(), true);
+        end_obj(false);
+        start_obj("offsets");
+        value_i("from", t0 * 10, false);
+        value_i("to", t1 * 10, true);
+        end_obj(end);
+    };
+
+    start_obj(nullptr);
+        value_s("systeminfo", whisper_print_system_info(), false);
+        start_obj("model");
+            value_s("type", whisper_model_type_readable(ctx), false);
+            value_b("multilingual", whisper_is_multilingual(ctx), false);
+            value_i("vocab", whisper_model_n_vocab(ctx), false);
+            start_obj("audio");
+                value_i("ctx", whisper_model_n_audio_ctx(ctx), false);
+                value_i("state", whisper_model_n_audio_state(ctx), false);
+                value_i("head", whisper_model_n_audio_head(ctx), false);
+                value_i("layer", whisper_model_n_audio_layer(ctx), true);
+            end_obj(false);
+            start_obj("text");
+                value_i("ctx", whisper_model_n_text_ctx(ctx), false);
+                value_i("state", whisper_model_n_text_state(ctx), false);
+                value_i("head", whisper_model_n_text_head(ctx), false);
+                value_i("layer", whisper_model_n_text_layer(ctx), true);
+            end_obj(false);
+            value_i("mels", whisper_model_n_mels(ctx), false);
+            value_i("ftype", whisper_model_ftype(ctx), true);
+        end_obj(false);
+        start_obj("result");
+            value_s("language", whisper_lang_str(whisper_full_lang_id(ctx)), true);
+        end_obj(false);
+        start_arr("transcription");
+
+        for (int i = first_segment; i < n_segments; ++i) {
+            const char * text = whisper_full_get_segment_text(ctx, i);
+
+            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
+            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
+
+            start_obj(nullptr);
+                times_o(t0, t1, false);
+                value_s("text", text, false);
+
+                start_arr("tokens");
+                const int n = whisper_full_n_tokens(ctx, i);
+                for (int j = 0; j < n; ++j) {
+                    auto token = whisper_full_get_token_data(ctx, i, j);
+                    start_obj(nullptr);
+                        value_s("text", whisper_token_to_str(ctx, token.id), false);
+                        if(token.t0 > -1 && token.t1 > -1) {
+                            // If we have per-token timestamps, write them out
+                            times_o(token.t0, token.t1, false);
+                        }
+                        value_i("id", token.id, false);
+                        value_f("p", token.p, false);
+                        value_f("t_dtw", token.t_dtw, true);
+                    end_obj(j == (n - 1));
+                }
+                end_arr(true);
+
+            end_obj(i == (n_segments - 1));
+        }
+
+        end_arr(true);
+    end_obj(true);
+
+    if (final) {
+        printf("remotion_final:%s\n", output.c_str());
+    } else {
+        printf("remotion_update:%s\n", output.c_str());
+    }
+
+    return true;
+}
+
+
+void whisper_print_segment_callback(struct whisper_context * ctx, struct whisper_state * /*state*/, int n_new, void * user_data) {
+    const int n_segments = whisper_full_n_segments(ctx);
+    const int s0 = n_segments - n_new;
+
+    if (s0 == 0) {
+        printf("\n");
+    }
+
+    output_json(ctx, false, s0, n_segments);
+}
+
+// Define the progress callback function
+void progress_callback(struct whisper_context * ctx, struct whisper_state * state, int progress, void * user_data) {    
+    printf("remotion_progress:%d%%\n", progress);
+}
+
+std::vector<struct whisper_context *> g_contexts(1, nullptr);
+
+
+EMSCRIPTEN_BINDINGS(whisper) {
+    emscripten::function("full_default", emscripten::optional_override([](const std::string & path_model, const emscripten::val & audio, const std::string & model, const std::string & lang, int nthreads, bool translate) {
+        if (g_contexts[0] != nullptr) {
+            printf("remotion_busy:\n");
+            return 0;
+        }
+
+        g_contexts[0] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());
+
+        struct whisper_full_params params = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);
+
+        std::vector<float> pcmf32;
+
+        params.print_realtime   = false;
+        params.new_segment_callback = whisper_print_segment_callback;
+        params.print_progress   = false;
+        params.print_timestamps = false;
+        params.print_special    = false;
+        params.translate        = translate;
+        params.token_timestamps = true;
+        params.language         = lang.c_str(); // Convert std::string to const char*
+        params.n_threads        = std::min(nthreads, std::min(16, mpow2(std::thread::hardware_concurrency())));
+        params.offset_ms        = 0;
+        params.progress_callback = progress_callback; // Assigning the callback
+
+        const int n = audio["length"].as<int>();
+
+        emscripten::val heap = emscripten::val::module_property("HEAPU8");
+        emscripten::val memory = heap["buffer"];
+
+        pcmf32.resize(n);
+
+        emscripten::val memoryView = audio["constructor"].new_(memory, reinterpret_cast<uintptr_t>(pcmf32.data()), n);
+        memoryView.call<void>("set", audio);
+
+        // Print system information
+        {
+            printf("system_info: n_threads = %d / %d | %s\n",
+                   params.n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());
+
+            printf("%s: processing %d samples, %.1f sec, %d threads, lang = %s, task = %s ...\n",
+                   __func__, int(pcmf32.size()), float(pcmf32.size()) / WHISPER_SAMPLE_RATE,
+                   params.n_threads,
+                   params.language,
+                   params.translate ? "translate" : "transcribe");
+
+            printf("\n");
+        }
+
+
+        // Run the worker
+        {
+            g_worker = std::thread([params, pcm = std::move(pcmf32)]() {
+                whisper_reset_timings(g_contexts[0]);
+                whisper_full(g_contexts[0], params, pcm.data(), pcm.size());
+                const int n_segments = whisper_full_n_segments(g_contexts[0]);
+                output_json(g_contexts[0], true, 0, n_segments);
+                whisper_free(g_contexts[0]);
+                g_contexts[0] = nullptr;
+            });
+        }
+
+        return 0;
+    }));
+}

package/eslint.config.mjs

@@ -0,0 +1,5 @@
+import {remotionFlatConfig} from '@remotion/eslint-config-internal';
+
+const config = remotionFlatConfig({react: false});
+
+export default [config];

package/main.d.ts

@@ -0,0 +1,46 @@
+// TypeScript bindings for emscripten-generated code.  Automatically generated at compile time.
+declare namespace RuntimeExports {
+    let print: any;
+    let printErr: any;
+    /**
+     * @param {string|null=} returnType
+     * @param {Array=} argTypes
+     * @param {Arguments|Array=} args
+     * @param {Object=} opts
+     */
+    function ccall(ident: any, returnType?: (string | null) | undefined, argTypes?: any[] | undefined, args?: (Arguments | any[]) | undefined, opts?: any | undefined): any;
+    /**
+     * @param {string=} returnType
+     * @param {Array=} argTypes
+     * @param {Object=} opts
+     */
+    function cwrap(ident: any, returnType?: string | undefined, argTypes?: any[] | undefined, opts?: any | undefined): (...args: any[]) => any;
+    let HEAPF32: any;
+    let HEAPF64: any;
+    let HEAP_DATA_VIEW: any;
+    let HEAP8: any;
+    let HEAPU8: any;
+    let HEAP16: any;
+    let HEAPU16: any;
+    let HEAP32: any;
+    let HEAPU32: any;
+    let HEAP64: any;
+    let HEAPU64: any;
+    let FS_createPath: any;
+    function FS_createDataFile(parent: any, name: any, fileData: any, canRead: any, canWrite: any, canOwn: any): void;
+    function FS_createPreloadedFile(parent: any, name: any, url: any, canRead: any, canWrite: any, onload: any, onerror: any, dontCreateFile: any, canOwn: any, preFinish: any): void;
+    function FS_unlink(path: any): any;
+    let FS_createLazyFile: any;
+    let FS_createDevice: any;
+    let addRunDependency: any;
+    let removeRunDependency: any;
+}
+interface WasmModule {
+}
+
+type EmbindString = ArrayBuffer|Uint8Array|Uint8ClampedArray|Int8Array|string;
+interface EmbindModule {
+  full_default(_0: EmbindString, _1: any, _2: EmbindString, _3: EmbindString, _4: number, _5: boolean): number;
+}
+
+export type MainModule = WasmModule & typeof RuntimeExports & EmbindModule;