whispercpp 1.3.1 → 1.3.3

data/ext/sources/tests/test-vad-full.cpp

@@ -0,0 +1,54 @@
+#include "whisper.h"
+#include "common-whisper.h"
+
+#include <cstdio>
+#include <cfloat>
+#include <string>
+#include <cstring>
+
+#ifdef NDEBUG
+#undef NDEBUG
+#endif
+
+#include <cassert>
+
+int main() {
+    std::string whisper_model_path = "../../models/ggml-base.en.bin";
+    std::string vad_model_path     = "../../models/for-tests-silero-v5.1.2-ggml.bin";
+    std::string sample_path        = "../../samples/jfk.wav";
+
+    // Load the sample audio file
+    std::vector<float> pcmf32;
+    std::vector<std::vector<float>> pcmf32s;
+    assert(read_audio_data(sample_path.c_str(), pcmf32, pcmf32s, false));
+
+    struct whisper_context_params cparams = whisper_context_default_params();
+    struct whisper_context * wctx = whisper_init_from_file_with_params(
+            whisper_model_path.c_str(),
+            cparams);
+
+    struct whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH);
+    wparams.vad            = true;
+    wparams.vad_model_path = vad_model_path.c_str();
+
+    wparams.vad_params.threshold               = 0.5f;
+    wparams.vad_params.min_speech_duration_ms  = 250;
+    wparams.vad_params.min_silence_duration_ms = 100;
+    wparams.vad_params.max_speech_duration_s   = FLT_MAX;
+    wparams.vad_params.speech_pad_ms           = 30;
+
+    assert(whisper_full_parallel(wctx, wparams, pcmf32.data(), pcmf32.size(), 1) == 0);
+
+    const int n_segments = whisper_full_n_segments(wctx);
+    assert(n_segments == 1);
+
+    assert(strcmp(" And so my fellow Americans, ask not what your country can do for you,"
+                  " ask what you can do for your country.",
+           whisper_full_get_segment_text(wctx, 0)) == 0);
+    assert(whisper_full_get_segment_t0(wctx, 0) == 29);
+    assert(whisper_full_get_segment_t1(wctx, 0) == 1049);
+
+    whisper_free(wctx);
+
+    return 0;
+}

data/ext/sources/tests/test-vad.cpp

@@ -0,0 +1,83 @@
+#include "whisper.h"
+#include "common-whisper.h"
+
+#include <cstdio>
+#include <string>
+
+#ifdef NDEBUG
+#undef NDEBUG
+#endif
+#include <cassert>
+
+void assert_default_params(const struct whisper_vad_params & params) {
+    assert(params.threshold == 0.5);
+    assert(params.min_speech_duration_ms == 250);
+    assert(params.min_silence_duration_ms == 100);
+    assert(params.samples_overlap == 0.1f);
+}
+
+void assert_default_context_params(const struct whisper_vad_context_params & params) {
+    assert(params.n_threads == 4);
+    assert(params.use_gpu == false);
+    assert(params.gpu_device == 0);
+}
+
+void test_detect_speech(
+        struct whisper_vad_context * vctx,
+        struct whisper_vad_params params,
+        const float * pcmf32,
+        int n_samples) {
+    assert(whisper_vad_detect_speech(vctx, pcmf32, n_samples));
+    assert(whisper_vad_n_probs(vctx) == 344);
+    assert(whisper_vad_probs(vctx) != nullptr);
+}
+
+struct whisper_vad_segments * test_detect_timestamps(
+        struct whisper_vad_context * vctx,
+        struct whisper_vad_params params) {
+    struct whisper_vad_segments * timestamps = whisper_vad_segments_from_probs(vctx, params);
+    assert(whisper_vad_segments_n_segments(timestamps) == 5);
+
+    for (int i = 0; i < whisper_vad_segments_n_segments(timestamps); ++i) {
+        printf("VAD segment %d: start = %.2f, end = %.2f\n", i,
+               whisper_vad_segments_get_segment_t0(timestamps, i),
+               whisper_vad_segments_get_segment_t1(timestamps, i));
+    }
+
+    return timestamps;
+}
+
+int main() {
+    std::string vad_model_path = "../../models/for-tests-silero-v5.1.2-ggml.bin";
+    std::string sample_path    = "../../samples/jfk.wav";
+
+    // Load the sample audio file
+    std::vector<float> pcmf32;
+    std::vector<std::vector<float>> pcmf32s;
+    assert(read_audio_data(sample_path.c_str(), pcmf32, pcmf32s, false));
+    assert(pcmf32.size() > 0);
+    assert(pcmf32s.size() == 0); // no stereo vector
+
+    // Load the VAD model
+    struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();
+    assert_default_context_params(ctx_params);
+
+    struct whisper_vad_context * vctx = whisper_vad_init_from_file_with_params(
+            vad_model_path.c_str(),
+            ctx_params);
+    assert(vctx != nullptr);
+
+    struct whisper_vad_params params = whisper_vad_default_params();
+    assert_default_params(params);
+
+    // Test speech probabilites
+    test_detect_speech(vctx, params, pcmf32.data(), pcmf32.size());
+
+    // Test speech timestamps (uses speech probabilities from above)
+    struct whisper_vad_segments * timestamps = test_detect_timestamps(vctx, params);
+
+    whisper_vad_free_segments(timestamps);
+    whisper_vad_free(vctx);
+
+    return 0;
+}

data/ext/sources/tests/test-whisper.js

@@ -0,0 +1,58 @@
+var factory = require('../bindings/javascript/whisper.js')
+
+factory().then(function(whisper) {
+    var fs = require('fs');
+
+    // to avoid reading WAV files and depending on some 3rd-party package, we read
+    // 32-bit float PCM directly. to genereate it:
+    //
+    //   $ ffmpeg -i samples/jfk.wav -f f32le -acodec pcm_f32le samples/jfk.pcmf32
+    //
+    let fname_wav   = "../samples/jfk.pcmf32";
+    let fname_model = "../models/ggml-base.en.bin";
+
+    // init whisper
+    {
+        // read binary data from file
+        var model_data = fs.readFileSync(fname_model);
+        if (model_data == null) {
+            console.log("whisper: failed to read model file");
+            process.exit(1);
+        }
+
+        // write binary data to WASM memory
+        whisper.FS_createDataFile("/", "whisper.bin", model_data, true, true);
+
+        // init the model
+        var ret = whisper.init("whisper.bin");
+        if (ret == false) {
+            console.log('whisper: failed to init');
+            process.exit(1);
+        }
+    }
+
+    // transcribe wav file
+    {
+        // read raw binary data
+        var pcm_data = fs.readFileSync(fname_wav);
+        if (pcm_data == null) {
+            console.log("whisper: failed to read wav file");
+            process.exit(1);
+        }
+
+        // convert to 32-bit float array
+        var pcm = new Float32Array(pcm_data.buffer);
+
+        // transcribe
+        var ret = whisper.full_default(pcm, "en", false);
+        if (ret != 0) {
+            console.log("whisper: failed to transcribe");
+            process.exit(1);
+        }
+    }
+
+    // free memory
+    {
+        whisper.free();
+    }
+});

data/extsources.rb

@@ -1,6 +1,40 @@
-require "yaml"
+require "pathname"
 
-sources = `git ls-files -z ../..`.split("\x0")
-paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"]
-paths.delete "bindings/ruby/**"
-EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources
+root = Pathname("..")/".."
+ignored_dirs = %w[
+  .devops
+  .github
+  ci
+  examples/wchess/wchess.wasm
+  examples/whisper.android
+  examples/whisper.android.java
+  examples/whisper.objc
+  examples/whisper.swiftui
+  grammars
+  models
+  samples
+  scripts
+].collect {|dir| root/dir}
+ignored_files = %w[
+  AUTHORS
+  Makefile
+  README.md
+  README_sycl.md
+  .gitignore
+  .gitmodules
+  .dockerignore
+  whisper.nvim
+  twitch.sh
+  yt-wsp.sh
+  close-issue.yml
+]
+
+EXTSOURCES =
+  `git ls-files -z #{root}`.split("\x0")
+    .collect {|file| Pathname(file)}
+    .reject {|file|
+      ignored_dirs.any? {|dir| file.descend.any? {|desc| desc == dir}} ||
+        ignored_files.include?(file.basename.to_path) ||
+        (file.descend.to_a[1] != root && file.descend.to_a[1] != Pathname("..")/"javascript")
+    }
+    .collect(&:to_path)

data/lib/whisper/context.rb

@@ -0,0 +1,15 @@
+module Whisper
+  class Context
+    def to_srt
+      each_segment.with_index.reduce("") {|srt, (segment, index)|
+        srt << "#{index + 1}\n#{segment.to_srt_cue}\n"
+      }
+    end
+
+    def to_webvtt
+      each_segment.with_index.reduce("WEBVTT\n\n") {|webvtt, (segment, index)|
+        webvtt << "#{index + 1}\n#{segment.to_webvtt_cue}\n"
+      }
+    end
+  end
+end

data/lib/whisper/model/uri.rb

@@ -1,157 +1,233 @@
-require "whisper.so"
 require "uri"
 require "net/http"
 require "time"
 require "pathname"
 require "io/console/size"
 
-class Whisper::Model
-  class URI
-    def initialize(uri)
-      @uri = URI(uri)
-    end
+module Whisper
+  class Model
+    class URI
+      def initialize(uri)
+        @uri = URI(uri)
+      end
 
-    def to_path
-      cache
-      cache_path.to_path
-    end
+      def to_path
+        cache
+        cache_path.to_path
+      end
 
-    def clear_cache
-      path = cache_path
-      path.delete if path.exist?
-    end
+      def clear_cache
+        path = cache_path
+        path.delete if path.exist?
+      end
 
-    private
+      private
 
-    def cache_path
-      base_cache_dir/@uri.host/@uri.path[1..]
-    end
+      def cache_path
+        base_cache_dir/@uri.host/@uri.path[1..]
+      end
 
-    def base_cache_dir
-      base = case RUBY_PLATFORM
-             when /mswin|mingw/
-               ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
-             when /darwin/
-               Pathname(Dir.home)/"Library/Caches"
-             else
-               ENV.key?("XDG_CACHE_HOME") ? ENV["XDG_CACHE_HOME"] : Pathname(Dir.home)/".cache"
-             end
-      base/"whisper.cpp"
-    end
+      def base_cache_dir
+        base = case RUBY_PLATFORM
+               when /mswin|mingw/
+                 ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
+               when /darwin/
+                 Pathname(Dir.home)/"Library/Caches"
+               else
+                 ENV.key?("XDG_CACHE_HOME") ? Pathname(ENV["XDG_CACHE_HOME"]) : Pathname(Dir.home)/".cache"
+               end
+        base/"whisper.cpp"
+      end
 
-    def cache
-      path = cache_path
-      headers = {}
-      headers["if-modified-since"] = path.mtime.httpdate if path.exist?
-      request @uri, headers
-      path
-    end
+      def cache
+        path = cache_path
+        headers = {}
+        headers["if-modified-since"] = path.mtime.httpdate if path.exist?
+        request @uri, headers
+        path
+      end
 
-    def request(uri, headers)
-      Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
-        request = Net::HTTP::Get.new(uri, headers)
-        http.request request do |response|
-          case response
-          when Net::HTTPNotModified
-            # noop
-          when Net::HTTPOK
-            download response
-          when Net::HTTPRedirection
-            request URI(response["location"]), headers
-          else
-            return if headers.key?("if-modified-since") # Use cache file
-
-            raise "#{response.code} #{response.message}\n#{response.body}"
+      def request(uri, headers)
+        Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
+          request = Net::HTTP::Get.new(uri, headers)
+          http.request request do |response|
+            case response
+            when Net::HTTPNotModified
+              # noop
+            when Net::HTTPOK
+              return if !response.key?("last-modified") && cache_path.exist?
+
+              download response
+            when Net::HTTPRedirection
+              request URI(response["location"]), headers
+            else
+              return if headers.key?("if-modified-since") # Use cache file
+
+              raise "#{response.code} #{response.message}\n#{response.body}"
+            end
           end
         end
+      rescue => err
+        if cache_path.exist?
+          warn err
+          # Use cache file
+        else
+          raise
+        end
       end
-    end
 
-    def download(response)
-      path = cache_path
-      path.dirname.mkpath unless path.dirname.exist?
-      downloading_path = Pathname("#{path}.downloading")
-      size = response.content_length
-      downloading_path.open "wb" do |file|
-        downloaded = 0
-        response.read_body do |chunk|
-          file << chunk
-          downloaded += chunk.bytesize
-          show_progress downloaded, size
+      def download(response)
+        path = cache_path
+        path.dirname.mkpath unless path.dirname.exist?
+        downloading_path = Pathname("#{path}.downloading")
+        size = response.content_length
+        downloading_path.open "wb" do |file|
+          downloaded = 0
+          response.read_body do |chunk|
+            file << chunk
+            downloaded += chunk.bytesize
+            show_progress downloaded, size
+          end
+          $stderr.puts
         end
+        downloading_path.rename path
       end
-      downloading_path.rename path
-    end
 
-    def show_progress(current, size)
-      return unless $stderr.tty?
-      return unless size
+      def show_progress(current, size)
+        progress_rate_available = size && $stderr.tty?
+
+        unless @prev
+          @prev = Time.now
+          $stderr.puts "Downloading #{@uri} to #{cache_path}"
+        end
+
+        now = Time.now
+
+        if progress_rate_available
+          return if now - @prev < 1 && current < size
 
-      unless @prev
-        @prev = Time.now
-        $stderr.puts "Downloading #{@uri}"
+          progress_width = 20
+          progress = current.to_f / size
+          arrow_length = progress * progress_width
+          arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
+          line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
+          padding = ' ' * ($stderr.winsize[1] - line.size)
+          $stderr.print "\r#{line}#{padding}"
+        else
+          return if now - @prev < 1
+
+          $stderr.print "."
+        end
+        @prev = now
       end
 
-      now = Time.now
-      return if now - @prev < 1 && current < size
-
-      progress_width = 20
-      progress = current.to_f / size
-      arrow_length = progress * progress_width
-      arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
-      line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
-      padding = ' ' * ($stderr.winsize[1] - line.size)
-      $stderr.print "\r#{line}#{padding}"
-      $stderr.puts if current >= size
-      @prev = now
+      def format_bytesize(bytesize)
+        return "0.0 B" if bytesize.zero?
+
+        units = %w[B KiB MiB GiB TiB]
+        exp = (Math.log(bytesize) / Math.log(1024)).to_i
+        format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
+      end
     end
 
-    def format_bytesize(bytesize)
-      return "0.0 B" if bytesize.zero?
+    class ZipURI < URI
+      def cache
+        zip_path = super
+        dest = unzipped_path
+        return if dest.exist? && dest.mtime >= zip_path.mtime
+        escaping dest do
+          system "unzip", "-q", "-d", zip_path.dirname.to_path, zip_path.to_path, exception: true
+        end
+        zip_path
+      end
+
+      def clear_cache
+        super
+        unzipped_path.rmtree if unzipped_path.exist?
+      end
+
+      private
+
+      def unzipped_path
+        cache_path.sub_ext("")
+      end
 
-      units = %w[B KiB MiB GiB TiB]
-      exp = (Math.log(bytesize) / Math.log(1024)).to_i
-      format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
+      def escaping(path)
+        escaped = Pathname("#{path}.removing")
+        if path.exist?
+          escaped.rmtree if escaped.exist?
+          path.rename escaped
+        end
+        yield
+      ensure
+        if path.exist?
+          escaped.rmtree if escaped.exist?
+        else
+          escaped.rename path if escaped.exist?
+        end
+      end
     end
-  end
 
-  @pre_converted_models = {}
-  %w[
-    tiny
-    tiny.en
-    tiny-q5_1
-    tiny.en-q5_1
-    tiny-q8_0
-    base
-    base.en
-    base-q5_1
-    base.en-q5_1
-    base-q8_0
-    small
-    small.en
-    small.en-tdrz
-    small-q5_1
-    small.en-q5_1
-    small-q8_0
-    medium
-    medium.en
-    medium-q5_0
-    medium.en-q5_0
-    medium-q8_0
-    large-v1
-    large-v2
-    large-v2-q5_0
-    large-v2-q8_0
-    large-v3
-    large-v3-q5_0
-    large-v3-turbo
-    large-v3-turbo-q5_0
-    large-v3-turbo-q8_0
-  ].each do |name|
-    @pre_converted_models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
-  end
+    @pre_converted_models = %w[
+      tiny
+      tiny.en
+      tiny-q5_1
+      tiny.en-q5_1
+      tiny-q8_0
+      base
+      base.en
+      base-q5_1
+      base.en-q5_1
+      base-q8_0
+      small
+      small.en
+      small.en-tdrz
+      small-q5_1
+      small.en-q5_1
+      small-q8_0
+      medium
+      medium.en
+      medium-q5_0
+      medium.en-q5_0
+      medium-q8_0
+      large-v1
+      large-v2
+      large-v2-q5_0
+      large-v2-q8_0
+      large-v3
+      large-v3-q5_0
+      large-v3-turbo
+      large-v3-turbo-q5_0
+      large-v3-turbo-q8_0
+    ].each_with_object({}) {|name, models|
+      models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
+    }
+
+    %w[
+      silero-v5.1.2
+    ].each do |name|
+      @pre_converted_models[name] = URI.new("https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-#{name}.bin")
+    end
 
-  class << self
-    attr_reader :pre_converted_models
+    @coreml_compiled_models = %w[
+      tiny
+      tiny.en
+      base
+      base.en
+      small
+      small.en
+      medium
+      medium.en
+      large-v1
+      large-v2
+      large-v3
+      large-v3-turbo
+    ].each_with_object({}) do |name, models|
+      models[@pre_converted_models[name]] = ZipURI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}-encoder.mlmodelc.zip")
+    end
+
+    class << self
+      attr_reader :pre_converted_models, :coreml_compiled_models
+    end
   end
 end

data/lib/whisper/segment.rb

@@ -0,0 +1,58 @@
+module Whisper
+  class Segment
+    SRT_ESCAPES = {
+      "&" => "&",
+      "<" => "&lt;",
+      ">" => "&gt;",
+    }
+    SRT_ESCAPES_RE = Regexp.union(SRT_ESCAPES.keys)
+    private_constant :SRT_ESCAPES, :SRT_ESCAPES_RE
+
+    def to_srt_cue
+      "#{srt_start_time} --> #{srt_end_time}\n#{srt_text}\n"
+    end
+
+    def to_webvtt_cue
+      "#{webvtt_start_time} --> #{webvtt_end_time}\n#{webvtt_text}\n"
+    end
+
+    private
+
+    def time_to_a(time)
+      sec, decimal_part = time.divmod(1000)
+      min, sec = sec.divmod(60)
+      hour, min = min.divmod(60)
+      [hour, min, sec, decimal_part]
+    end
+
+    def srt_time(time)
+      "%02d:%02d:%02d,%03d" % time_to_a(time)
+    end
+
+    def srt_start_time
+      srt_time(start_time)
+    end
+
+    def srt_end_time
+      srt_time(end_time)
+    end
+
+    def srt_text
+      text.gsub(SRT_ESCAPES_RE, SRT_ESCAPES)
+    end
+
+    def webvtt_time(time)
+      "%02d:%02d:%02d.%03d" % time_to_a(time)
+    end
+
+    def webvtt_start_time
+      webvtt_time(start_time)
+    end
+
+    def webvtt_end_time
+      webvtt_time(end_time)
+    end
+
+    alias webvtt_text srt_text
+  end
+end