whispercpp 1.3.5 → 1.3.7

data/ext/sources/src/whisper.cpp

@@ -3720,7 +3720,21 @@ struct whisper_context * whisper_init_with_params_no_state(struct whisper_model_
     whisper_context * ctx = new whisper_context;
     ctx->params = params;
 
-    if (!whisper_model_load(loader, *ctx)) {
+    // A C++ exception escaping this extern "C" function aborts non-C++ callers
+    // (Rust via whisper-rs, Go via cgo, ...). whisper_model_load can throw
+    // (std::runtime_error here; vk::SystemError from the Vulkan backend during
+    // device/buffer allocation), so funnel any throw into the existing
+    // NULL-return failure path instead of letting it cross the C ABI.
+    bool model_loaded = false;
+    try {
+        model_loaded = whisper_model_load(loader, *ctx);
+    } catch (const std::exception & e) {
+        WHISPER_LOG_ERROR("%s: exception during model load: %s\n", __func__, e.what());
+    } catch (...) {
+        WHISPER_LOG_ERROR("%s: unknown exception during model load\n", __func__);
+    }
+
+    if (!model_loaded) {
         loader->close(loader->context);
         WHISPER_LOG_ERROR("%s: failed to load model\n", __func__);
         delete ctx;
@@ -5083,7 +5097,11 @@ struct whisper_vad_context * whisper_vad_init_with_params(
     return vctx;
 }
 
-bool whisper_vad_detect_speech(
+void whisper_vad_reset_state(whisper_vad_context * vctx) {
+    ggml_backend_buffer_clear(vctx->buffer, 0);
+}
+
+bool whisper_vad_detect_speech_no_reset(
         struct whisper_vad_context * vctx,
         const float * samples,
         int n_samples) {
@@ -5095,9 +5113,6 @@ bool whisper_vad_detect_speech(
     WHISPER_LOG_INFO("%s: detecting speech in %d samples\n", __func__, n_samples);
     WHISPER_LOG_INFO("%s: n_chunks: %d\n", __func__, n_chunks);
 
-    // Reset LSTM hidden/cell states
-    ggml_backend_buffer_clear(vctx->buffer, 0);
-
     vctx->probs.resize(n_chunks);
     WHISPER_LOG_INFO("%s: props size: %u\n", __func__, n_chunks);
 
@@ -5165,6 +5180,14 @@ bool whisper_vad_detect_speech(
     return true;
 }
 
+bool whisper_vad_detect_speech(
+        struct whisper_vad_context * vctx,
+        const float * samples,
+        int n_samples) {
+    whisper_vad_reset_state(vctx);
+    return whisper_vad_detect_speech_no_reset(vctx, samples, n_samples);
+}
+
 int whisper_vad_segments_n_segments(struct whisper_vad_segments * segments) {
     return segments->data.size();
 }
@@ -6207,6 +6230,13 @@ static void whisper_process_logits(
             }
         }
 
+        // ref: https://github.com/ggml-org/whisper.cpp/pull/3798
+        if (!params.no_timestamps && !params.single_segment && params.max_tokens > 0 && (int) tokens_cur.size() >= params.max_tokens) {
+            for (int i = 0; i < vocab.token_eot; ++i) {
+                logits[i] = -INFINITY;
+            }
+        }
+
         // suppress sot and nosp tokens
         logits[vocab.token_sot]  = -INFINITY;
         logits[vocab.token_nosp] = -INFINITY;
@@ -6701,12 +6731,13 @@ static bool whisper_vad(
             int segment_start_samples = cs_to_samples(vad_segments->data[i].start);
             int segment_end_samples   = cs_to_samples(vad_segments->data[i].end);
 
-            if (i < (int)vad_segments->data.size() - 1) {
-                segment_end_samples += overlap_samples;
-            }
-
             segment_start_samples = std::min(segment_start_samples, n_samples - 1);
             segment_end_samples = std::min(segment_end_samples, n_samples - 1);
+            int original_segment_length = segment_end_samples - segment_start_samples;
+
+            if (i < (int)vad_segments->data.size() - 1) {
+                segment_end_samples = std::min(segment_end_samples + overlap_samples, n_samples - 1);
+            }
             int segment_length = segment_end_samples - segment_start_samples;
             if (segment_length > 0) {
                 whisper_state::vad_segment_info segment;
@@ -6715,7 +6746,7 @@ static bool whisper_vad(
                 segment.orig_end   = vad_segments->data[i].end;
 
                 segment.vad_start = samples_to_cs(offset);
-                segment.vad_end   = samples_to_cs(offset + segment_length);
+                segment.vad_end   = samples_to_cs(offset + original_segment_length);
 
                 // Add segment boundaries to mapping table
                 vad_time_mapping start_mapping = {segment.vad_start, segment.orig_start};
@@ -6724,29 +6755,6 @@ static bool whisper_vad(
                 state->vad_mapping_table.push_back(start_mapping);
                 state->vad_mapping_table.push_back(end_mapping);
 
-                // Add intermediate points for longer segments to improve interpolation accuracy
-                const int64_t min_segment_length = 100; // 1 second
-                const int64_t point_interval = 20;     // Add a point every 200ms
-
-                if (segment.vad_end - segment.vad_start > min_segment_length) {
-                    int64_t segment_duration = segment.vad_end - segment.vad_start;
-                    int num_points = (int)(segment_duration / point_interval) - 1;
-
-                    for (int j = 1; j <= num_points; j++) {
-                        int64_t vad_time = segment.vad_start + j * point_interval;
-
-                        if (vad_time >= segment.vad_end) continue;
-
-                        int64_t vad_elapsed = vad_time - segment.vad_start;
-                        int64_t vad_total = segment.vad_end - segment.vad_start;
-                        int64_t orig_total = segment.orig_end - segment.orig_start;
-                        int64_t orig_time = segment.orig_start + (vad_elapsed * orig_total) / vad_total;
-
-                        vad_time_mapping intermediate_mapping = {vad_time, orig_time};
-                        state->vad_mapping_table.push_back(intermediate_mapping);
-                    }
-                }
-
                 WHISPER_LOG_INFO("%s: vad_segment_info: orig_start: %.2f, orig_end: %.2f, vad_start: %.2f, vad_end: %.2f\n",
                     __func__, segment.orig_start/100.0, segment.orig_end/100.0, segment.vad_start/100.0, segment.vad_end/100.0);
                 ctx->state->vad_segments.push_back(segment);
@@ -7672,11 +7680,14 @@ int whisper_full_with_state(
                             }
                         }
                         text = "";
-                        while (i < (int) tokens_cur.size() && tokens_cur[i].id > whisper_token_beg(ctx)) {
+                        t0 = t1;
+                        while (i + 1 < (int) tokens_cur.size() && tokens_cur[i + 1].id > whisper_token_beg(ctx)) {
                             i++;
+                            if (params.print_special) {
+                                text += whisper_token_to_str(ctx, tokens_cur[i].id);
+                            }
+                            t0 = seek + 2 * (tokens_cur[i].tid - whisper_token_beg(ctx));
                         }
-                        i--;
-                        t0 = t1;
                         i0 = i + 1;
                         speaker_turn_next = false;
                     }
@@ -7693,8 +7704,8 @@ int whisper_full_with_state(
                             printf("[%s --> %s]  %s\n", to_timestamp(tt0).c_str(), to_timestamp(tt1).c_str(), text.c_str());
                         } else {
                             printf("%s", text.c_str());
-                            fflush(stdout);
                         }
+                        fflush(stdout);
                     }
 
                     result_all.push_back({ tt0, tt1, text, state->no_speech_prob, {}, speaker_turn_next });
@@ -7735,7 +7746,12 @@ int whisper_full_with_state(
             }
 
             // ref: https://github.com/ggml-org/whisper.cpp/pull/2629
+            const bool max_tokens_timestamp_ending = params.max_tokens > 0 &&
+                !params.single_segment &&
+                tokens_cur.size() > (size_t) params.max_tokens;
+
             const bool single_timestamp_ending = tokens_cur.size() > 1 &&
+                !max_tokens_timestamp_ending &&
                 tokens_cur[tokens_cur.size() - 2].id < whisper_token_beg(ctx) &&
                 tokens_cur[tokens_cur.size() - 1].id > whisper_token_beg(ctx);
             if (single_timestamp_ending) {
@@ -8256,9 +8272,6 @@ WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) {
     // when F16 is used, there is an extra work buffer of size N*N*sizeof(float)
     std::vector<uint8_t> buf(3llu*N_max*N_max*sizeof(float) + 3*ggml_tensor_overhead() + ggml_graph_overhead());
 
-    // put a bunch of random data in the buffer
-    for (size_t i = 0; i < buf.size(); i++) buf[i] = i;
-
     for (int j = 0; j < (int) sizes.size(); j++) {
         int n_q4_0 = 0;
         int n_q4_1 = 0;
@@ -8302,6 +8315,15 @@ WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) {
             struct ggml_tensor * a = ggml_new_tensor_2d(ctx0, wtype,         N, N);
             struct ggml_tensor * b = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, N, N);
 
+            // set tensor data after allocation so previous iteration results don't corrupt it.
+            {
+                uint8_t * a_data = (uint8_t *) a->data;
+                for (size_t ii = 0; ii < ggml_nbytes(a); ii++) a_data[ii] = ii & 0x3F;
+
+                uint8_t * b_data = (uint8_t *) b->data;
+                for (size_t ii = 0; ii < ggml_nbytes(b); ii++) b_data[ii] = ii & 0x3F;
+            }
+
             struct ggml_tensor * c = ggml_mul_mat(ctx0, a, b);
 
             struct ggml_cgraph * gf = ggml_new_graph(ctx0);

data/extsources.rb

@@ -5,37 +5,53 @@ ignored_dirs = %w[
   .devops
   .github
   ci
-  examples/wchess/wchess.wasm
+  examples/addon.node
+  examples/bench.wasm
+  examples/command
+  examples/command.wasm
+  examples/lsp
+  examples/main
+  examples/python
+  examples/stream
+  examples/stream.wasm
+  examples/sycl
+  examples/talk-llama
+  examples/wchess
   examples/whisper.android
   examples/whisper.android.java
+  examples/whisper.nvim
   examples/whisper.objc
   examples/whisper.swiftui
+  examples/whisper.wasm
   grammars
   models
   samples
   scripts
+  tests
 ].collect {|dir| root/dir}
 ignored_files = %w[
   AUTHORS
   Makefile
-  README.md
-  README_sycl.md
   .gitignore
   .gitmodules
   .dockerignore
-  whisper.nvim
-  twitch.sh
-  yt-wsp.sh
-  close-issue.yml
-  build-xcframework.sh
+]
+ignored_exts = %w[
+  .yml
+  .sh
+  .md
+  .py
+  .js
+  .nvim
 ]
 
 EXTSOURCES =
   `git ls-files -z #{root}`.split("\x0")
     .collect {|file| Pathname(file)}
     .reject {|file|
-      ignored_dirs.any? {|dir| file.descend.any? {|desc| desc == dir}} ||
+      ignored_exts.include?(file.extname) ||
         ignored_files.include?(file.basename.to_path) ||
-        (file.descend.to_a[1] != root && file.descend.to_a[1] != Pathname("..")/"javascript")
+        ignored_dirs.any? {|dir| file.descend.any? {|desc| desc == dir}} ||
+        (file.descend.to_a[1] != root && file != Pathname("..")/"javascript"/"package-tmpl.json")
     }
     .collect(&:to_path)

data/lib/whisper/log_settable.rb

@@ -0,0 +1,36 @@
+require "mutex_m"
+
+module Whisper
+  module LogSettable
+    class << self
+      def extended(base)
+        base.extend Mutex_m
+      end
+    end
+
+    private
+
+    def start_log_callback_thread
+      return if @log_callback_thread&.alive?
+
+      @log_callback_thread = Thread.new {
+        begin
+          while logs = drain_logs
+            begin
+              callback, user_data = synchronize {[@log_callback, @log_callback_user_data]}
+              next if callback.nil?
+
+              logs.each do |(level, text)|
+                callback.call level, text, user_data
+              end
+            rescue => err
+              $stderr.puts err
+            end
+          end
+        rescue => err
+          $stderr.puts err
+        end
+      }
+    end
+  end
+end

data/lib/whisper/model/uri.rb

@@ -41,6 +41,8 @@ module Whisper
 
       def cache
         path = cache_path
+        return path if cache_path.exist?
+
         headers = {}
         headers["if-modified-since"] = path.mtime.httpdate if path.exist?
         request @uri, headers
@@ -216,8 +218,18 @@ module Whisper
       @pre_converted_models[name] = URI.new("https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-#{name}.bin")
     end
 
+    %w[
+      parakeet-tdt-0.6b-v3-f16
+      parakeet-tdt-0.6b-v3-f32
+      parakeet-tdt-0.6b-v3-q4_0
+      parakeet-tdt-0.6b-v3-q4_k
+      parakeet-tdt-0.6b-v3-q8_0
+    ].each do |name|
+      @pre_converted_models[name] = URI.new("https://huggingface.co/ggml-org/parakeet-GGUF/resolve/main/ggml-#{name}.bin")
+    end
+
     @coreml_compiled_models = @pre_converted_models.each_with_object({}) {|(name, uri), models|
-      next if name.end_with?("-tdrz") || name.start_with?("silero-")
+      next if name.end_with?("-tdrz") || name.start_with?("silero-") || name.start_with?("parakeet-")
 
       if matched = name.match(/\A(?<name>.*)-q\d_\d\z/)
         name = matched[:name]

data/lib/whisper/output.rb

@@ -0,0 +1,74 @@
+module Whisper
+  module Output
+    module Context
+      def to_srt
+        each_segment.with_index.reduce("") {|srt, (segment, index)|
+          srt << "#{index + 1}\n#{segment.to_srt_cue}\n"
+        }
+      end
+
+      def to_webvtt
+        each_segment.with_index.reduce("WEBVTT\n\n") {|webvtt, (segment, index)|
+          webvtt << "#{index + 1}\n#{segment.to_webvtt_cue}\n"
+        }
+      end
+    end
+
+    module Segment
+      SRT_ESCAPES = {
+        "&" => "&amp;",
+        "<" => "&lt;",
+        ">" => "&gt;",
+      }
+      SRT_ESCAPES_RE = Regexp.union(SRT_ESCAPES.keys)
+      private_constant :SRT_ESCAPES, :SRT_ESCAPES_RE
+
+      def to_srt_cue
+        "#{srt_start_time} --> #{srt_end_time}\n#{srt_text}\n"
+      end
+
+      def to_webvtt_cue
+        "#{webvtt_start_time} --> #{webvtt_end_time}\n#{webvtt_text}\n"
+      end
+
+      private
+
+      def time_to_a(time)
+        sec, decimal_part = time.divmod(1000)
+        min, sec = sec.divmod(60)
+        hour, min = min.divmod(60)
+        [hour, min, sec, decimal_part]
+      end
+
+      def srt_time(time)
+        "%02d:%02d:%02d,%03d" % time_to_a(time)
+      end
+
+      def srt_start_time
+        srt_time(start_time)
+      end
+
+      def srt_end_time
+        srt_time(end_time)
+      end
+
+      def srt_text
+        text.gsub(SRT_ESCAPES_RE, SRT_ESCAPES)
+      end
+
+      def webvtt_time(time)
+        "%02d:%02d:%02d.%03d" % time_to_a(time)
+      end
+
+      def webvtt_start_time
+        webvtt_time(start_time)
+      end
+
+      def webvtt_end_time
+        webvtt_time(end_time)
+      end
+
+      alias webvtt_text srt_text
+    end
+  end
+end