whispercpp 1.3.4 → 1.3.6

data/sig/whisper.rbs

@@ -17,6 +17,21 @@ module Whisper
   LOG_LEVEL_ERROR: Integer
   LOG_LEVEL_DEBUG: Integer
   LOG_LEVEL_CONT: Integer
+  AHEADS_NONE: Integer
+  AHEADS_N_TOP_MOST: Integer
+  AHEADS_CUSTOM: Integer
+  AHEADS_TINY_EN: Integer
+  AHEADS_TINY: Integer
+  AHEADS_BASE_EN: Integer
+  AHEADS_BASE: Integer
+  AHEADS_SMALL_EN: Integer
+  AHEADS_SMALL: Integer
+  AHEADS_MEDIUM_EN: Integer
+  AHEADS_MEDIUM: Integer
+  AHEADS_LARGE_V1: Integer
+  AHEADS_LARGE_V2: Integer
+  AHEADS_LARGE_V3: Integer
+  AHEADS_LARGE_V3_TURBO: Integer
 
   def self.lang_max_id: () -> Integer
   def self.lang_id: (string name) -> Integer
@@ -37,8 +52,8 @@ module Whisper
     #       puts text
     #     end
     #
-    def transcribe: (string, Params, ?n_processors: Integer) -> self
-                  | (string, Params, ?n_processors: Integer) { (String) -> void } -> self
+    def transcribe: (path, Params, ?n_processors: Integer) -> self
+                  | (path, Params, ?n_processors: Integer) { (String) -> void } -> self
 
     def model_n_vocab: () -> Integer
     def model_n_audio_ctx: () -> Integer
@@ -120,6 +135,30 @@ module Whisper
 
     def to_srt: () -> String
     def to_webvtt: () -> String
+
+    class Params
+      def self.new: (
+        use_gpu: boolish,
+        flash_attn: boolish,
+        gpu_device: Integer,
+        dtw_token_timestamps: boolish,
+        dtw_aheads_preset: Integer,
+        dtw_n_top: Integer | nil,
+      ) -> instance
+
+      def use_gpu=: (boolish) -> boolish
+      def use_gpu: () -> (true | false)
+      def flash_attn=: (boolish) -> boolish
+      def flash_attn: () -> (true | false)
+      def gpu_device=: (Integer) -> Integer
+      def gpu_device: () -> Integer
+      def dtw_token_timestamps=: (boolish) -> boolish
+      def dtw_token_timestamps: () -> (true | false)
+      def dtw_aheads_preset=: (Integer) -> Integer
+      def dtw_aheads_preset: () -> Integer
+      def dtw_n_top=: (Integer | nil) -> (Integer | nil)
+      def dtw_n_top: () -> (Integer | nil)
+    end
   end
 
   class Params
@@ -138,6 +177,7 @@ module Whisper
       ?max_len: Integer,
       ?split_on_word: boolish,
       ?initial_prompt: string | nil,
+      ?carry_initial_prompt: boolish,
       ?diarize: boolish,
       ?offset: Integer,
       ?duration: Integer,
@@ -236,6 +276,7 @@ module Whisper
     def split_on_word: () -> (true | false)
 
     def initial_prompt=: (_ToS) -> _ToS
+    def carry_initial_prompt=: (boolish) -> boolish
 
     # Tokens to provide to the whisper decoder as initial prompt
     # these are prepended to any existing text context from a previous call
@@ -243,6 +284,7 @@ module Whisper
     # Maximum of whisper_n_text_ctx()/2 tokens are used (typically 224).
     #
     def initial_prompt: () -> (String | nil)
+    def carry_initial_prompt: () -> (true | false)
 
     def diarize=: (boolish) -> boolish
 
@@ -431,7 +473,8 @@ module Whisper
       end_time: (Integer | nil),
       text: (String | nil),
       no_speech_prob: (Float | nil),
-      speaker_turn_next: (true | false | nil)
+      speaker_turn_next: (true | false | nil),
+      n_tokens: (Integer | nil)
     }
 
     # Start time in milliseconds.
@@ -443,13 +486,32 @@ module Whisper
     def end_time: () -> Integer
 
     # Whether the next segment is predicted as a speaker turn.
+    #
     def speaker_turn_next?: () -> (true | false)
 
     def text: () -> String
     def no_speech_prob: () -> Float
+
+    # Get number of tokens in the segment
+    #
+    def n_tokens: () -> Integer
+
+    # Yields each Whisper::Token:
+    #
+    #   whisper.each_segment.first.each_token do |token|
+    #     p token
+    #   end
+    #
+    # Returns an Enumerator if no block is given:
+    #
+    #   whisper.each_segment.first.each_token.to_a # => [#<Whisper::Token>, ...]
+    #
+    def each_token: { (Token) -> void } -> void
+                  | () -> Enumerator[Token]
     def to_srt_cue: () -> String
     def to_webvtt_cue: () -> String
 
+
     #  Possible keys: :start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next
     #
     #      whisper.each_segment do |segment|
@@ -457,7 +519,77 @@ module Whisper
     #
     #        puts "[#{start_time} --> #{end_time}] #{text} (no speech prob: #{no_speech_prob}#{speaker_turn_next ? ', speaker turns next' : ''})"
     #      end
-    def deconstruct_keys: (Array[:start_time | :end_time | :text | :no_speech_prob | :speaker_turn_next] | nil) -> deconstructed_keys
+    def deconstruct_keys: (Array[:start_time | :end_time | :text | :no_speech_prob | :speaker_turn_next | :n_tokens] | nil) -> deconstructed_keys
+  end
+
+  module Token
+    type deconstructed_keys = {
+      id: (Integer | nil),
+      tid: (Integer | nil),
+      probability: (Float | nil),
+      log_probability: (Float | nil),
+      pt: (Float | nil),
+      ptsum: (Float | nil),
+      t_dtw: (Integer | nil),
+      voice_length: (Float | nil),
+      text: (String | nil),
+      start_time: (Integer | nil),
+      end_time: (Integer | nil),
+    }
+
+    # Token ID.
+    #
+    def id: () -> Integer
+
+    # Forced timestamp token ID.
+    #
+    def tid: () -> Integer
+
+    # Probability of the token.
+    #
+    def probability: () -> Float
+
+    # Log probability of the token.
+    #
+    def log_probability: () -> Float
+
+    # Probability of the timestamp token.
+    #
+    def pt: () -> Float
+
+    # Sum of probability of all timestamp tokens.
+    #
+    def ptsum: () -> Float
+
+    # [EXPERIMENTAL] Token-level timestamps with DTW
+    #
+    # Do not use if you haven't computed token-level timestamps with dtw.
+    # Roughly corresponds to the moment in audio in which the token was output.
+    #
+    def t_dtw: () -> Integer
+
+    # Voice length of the token.
+    #
+    def voice_length: () -> Float
+
+    # Start time of the token.
+    #
+    # Token-level timestamp data.
+    # Do not use if you haven't computed token-level timestamps.
+    #
+    def start_time: () -> Integer
+
+    # End time of the token.
+    #
+    # Token-level timestamp data.
+    # Do not use if you haven't computed token-level timestamps.
+    #
+    def end_time: () -> Integer
+
+    # Get the token text of the token.
+    #
+    def text: () -> String
+    def deconstruct_keys: (Array[:id | :tid | :probability | :log_probability | :pt | :ptsum | :t_dtw | :voice_length | :start_time | :end_time | :text] | nil) -> deconstructed_keys
   end
 
   module VAD
@@ -507,6 +639,32 @@ module Whisper
       def samples_overlap: () -> Float
       def ==: (Params) -> (true | false)
     end
+
+    class Context
+      def self.new: (String | path | ::URI::HTTP model_name_or_path) -> instance
+      def segments_from_samples: (Params, Array[Float] samples, ?Integer n_samples) -> Segments
+                               | (Params, _Samples, ?Integer n_samples) -> Segments
+      def detect: (path wav_file_path, Params) -> Segments
+    end
+
+    class Segments
+      include Enumerable[Segment]
+
+      def each: { (Segment) -> void } -> void
+              | () -> Enumerator[Segment]
+      def length: -> Integer
+    end
+
+    class Segment
+      type deconstructed_keys = {
+        start_time: (Integer | nil),
+        end_time: (Integer | nil),
+      }
+
+      def start_time: () -> Integer
+      def end_time: () -> Integer
+      def deconstruct_keys: (Array[:start_time | :end_time] | nil) -> deconstructed_keys
+    end
   end
 
   class Error < StandardError

data/test/test_context_params.rb

@@ -0,0 +1,82 @@
+require_relative "helper"
+
+class TestContextParams < TestBase
+  PARAM_NAMES = [
+    :use_gpu,
+    :flash_attn,
+    :gpu_device,
+    :dtw_token_timestamps,
+    :dtw_aheads_preset,
+    :dtw_n_top
+  ]
+
+  def test_new
+    params = Whisper::Context::Params.new
+    assert_instance_of Whisper::Context::Params, params
+  end
+
+  def test_attributes
+    params = Whisper::Context::Params.new
+
+    assert_true params.use_gpu
+    params.use_gpu = false
+    assert_false params.use_gpu
+
+    assert_true params.flash_attn
+    params.flash_attn = false
+    assert_false params.flash_attn
+
+    assert_equal 0, params.gpu_device
+    params.gpu_device = 1
+    assert_equal 1, params.gpu_device
+
+    assert_false params.dtw_token_timestamps
+    params.dtw_token_timestamps = true
+    assert_true params.dtw_token_timestamps
+
+    assert_equal Whisper::AHEADS_NONE, params.dtw_aheads_preset
+    params.dtw_aheads_preset =Whisper::AHEADS_BASE
+    assert_equal Whisper::AHEADS_BASE, params.dtw_aheads_preset
+
+    assert_nil params.dtw_n_top
+    params.dtw_n_top = 6
+    assert_equal 6, params.dtw_n_top
+    params.dtw_n_top = nil
+    assert_nil params.dtw_n_top
+  end
+
+  def test_new_with_kw_args
+    params = Whisper::Context::Params.new(use_gpu: false)
+    assert_false params.use_gpu
+  end
+
+  def test_new_with_kw_wargs_non_existent
+    assert_raise ArgumentError do
+      Whisper::Context::Params.new(non_existent: "value")
+    end
+  end
+
+  data(PARAM_NAMES.collect {|param| [param, param]}.to_h)
+  def test_new_with_kw_args_default_values(param)
+    default_params = Whisper::Context::Params.new
+    default_value = default_params.send(param)
+    value = if param == :dtw_n_top
+              6
+            else
+              case default_value
+              in true | false
+                !default_value
+              in Integer
+                default_value + 1
+              end
+            end
+    params = Whisper::Context::Params.new(param => value)
+    assert_equal value, params.send(param)
+
+    PARAM_NAMES.reject {|name| name == param}.each do |name|
+      expected = default_params.send(name)
+      actual = params.send(name)
+      assert_equal expected, actual
+    end
+  end
+end

data/test/test_params.rb

@@ -16,6 +16,7 @@ class TestParams < TestBase
     :max_len,
     :split_on_word,
     :initial_prompt,
+    :carry_initial_prompt,
     :diarize,
     :offset,
     :duration,
@@ -119,6 +120,13 @@ class TestParams < TestBase
     assert !@params.print_timestamps
   end
 
+  def test_carry_initial_prompt
+    @params.carry_initial_prompt = true
+    assert @params.carry_initial_prompt
+    @params.carry_initial_prompt = false
+    assert !@params.carry_initial_prompt
+  end
+
   def test_suppress_blank
     @params.suppress_blank = true
     assert @params.suppress_blank
@@ -210,12 +218,12 @@ class TestParams < TestBase
 
   def test_vad_model_path
     assert_nil @params.vad_model_path
-    @params.vad_model_path = "silero-v5.1.2"
-    assert_equal Whisper::Model.pre_converted_models["silero-v5.1.2"].to_path, @params.vad_model_path
+    @params.vad_model_path = "silero-v6.2.0"
+    assert_equal Whisper::Model.pre_converted_models["silero-v6.2.0"].to_path, @params.vad_model_path
   end
 
   def test_vad_model_path_with_nil
-    @params.vad_model_path = "silero-v5.1.2"
+    @params.vad_model_path = "silero-v6.2.0"
     @params.vad_model_path = nil
     assert_nil @params.vad_model_path
   end
@@ -227,13 +235,13 @@ class TestParams < TestBase
   end
 
   def test_vad_model_path_with_URI_string
-    @params.vad_model_path = "https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v5.1.2.bin"
-    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models["silero-v5.1.2"].to_path
+    @params.vad_model_path = "https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin"
+    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models["silero-v6.2.0"].to_path
   end
 
   def test_vad_model_path_with_URI
-    @params.vad_model_path = URI("https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v5.1.2.bin")
-    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models["silero-v5.1.2"].to_path
+    @params.vad_model_path = URI("https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin")
+    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models["silero-v6.2.0"].to_path
   end
 
   def test_vad_params
@@ -281,7 +289,7 @@ class TestParams < TestBase
             in [/_user_data\Z/, *]
               Object.new
             in [:vad_model_path, *]
-              Whisper::Model.pre_converted_models["silero-v5.1.2"].to_path
+              Whisper::Model.pre_converted_models["silero-v6.2.0"].to_path
             in [:vad_params, *]
               Whisper::VAD::Params.new
             end

data/test/test_segment.rb

@@ -73,7 +73,6 @@ class TestSegment < TestBase
   end
 
   def test_transcription_after_segment_retrieved
-    params = Whisper::Params.new
     segment = whisper.each_segment.first
     assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)
 

data/test/test_token.rb

@@ -0,0 +1,81 @@
+require_relative "helper"
+
+class TestToken < TestBase
+  def setup
+    @segment = whisper.each_segment.first
+    @token = @segment.each_token.first
+  end
+
+  def test_n_tokens
+    assert_equal 27, @segment.n_tokens
+  end
+
+  def test_allocate
+    token = Whisper::Token.allocate
+    assert_raise  do
+      token.id
+    end
+  end
+
+  def test_each_token
+    i = 0
+    @segment.each_token do |token|
+      i += 1
+      assert_instance_of Whisper::Token, token
+    end
+    assert_equal 27, i
+  end
+
+  def test_each_token_without_block
+    assert_instance_of Enumerator, @segment.each_token
+  end
+
+  def test_token
+    assert_instance_of Whisper::Token, @token
+
+    assert_instance_of Integer, @token.id
+    assert_instance_of Float, @token.probability
+    assert_instance_of Float, @token.log_probability
+
+    assert_instance_of Integer, @token.tid
+    assert_instance_of Float, @token.pt
+    assert_instance_of Float, @token.ptsum
+
+    assert_instance_of Integer, @token.start_time
+    assert_instance_of Integer, @token.end_time
+
+    assert_instance_of Integer, @token.t_dtw
+
+    assert_instance_of Float, @token.voice_length
+
+    assert_instance_of String, @token.text
+  end
+
+  def test_text
+    assert_equal ["[_BEG_]", " And", " so", " my", " fellow", " Americans", ",", " ask", " not", " what", " your", " country", " can", " do", " for", " you", ",", " ask", " what", " you", " can", " do", " for", " your", " country", ".", "[_TT_550]"],
+                 @segment.each_token.collect(&:text)
+  end
+
+  def test_token_timestamps
+    params = Whisper::Params.new(token_timestamps: true)
+    whisper.transcribe(TestBase::AUDIO, params)
+    prev = -1
+    whisper.each_segment.first.each_token do |token|
+      assert token.start_time >= prev
+      assert token.end_time >= token.start_time
+      prev = token.end_time
+    end
+  end
+
+  def test_deconstruct_keys_with_nil
+    keys = %i[id tid probability log_probability pt ptsum t_dtw voice_length start_time end_time text]
+    expected = keys.collect {|key| [key, @token.send(key)] }.to_h
+    assert_equal(expected, @token.deconstruct_keys(nil))
+  end
+
+  def test_deconstruct_keys_with_keys
+    keys = %i[id tid probability log_probability pt ptsum t_dtw voice_length start_time end_time text]
+    expected = keys.collect {|key| [key, @token.send(key)] }.to_h
+    assert_equal expected, @token.deconstruct_keys(keys)
+  end
+end

data/test/test_vad.rb

@@ -6,7 +6,7 @@ class TestVAD < TestBase
     vad_params = Whisper::VAD::Params.new
     @params = Whisper::Params.new(
       vad: true,
-      vad_model_path: "silero-v5.1.2",
+      vad_model_path: "silero-v6.2.0",
       vad_params:
     )
   end

data/test/test_vad_context.rb

@@ -0,0 +1,100 @@
+require_relative "helper"
+
+class TestVADContext < TestBase
+  def test_initialize
+    context = Whisper::VAD::Context.new("silero-v6.2.0")
+    assert_instance_of Whisper::VAD::Context, context
+  end
+
+  def test_detect
+    context = Whisper::VAD::Context.new("silero-v6.2.0")
+    segments = context.detect(AUDIO, Whisper::VAD::Params.new)
+    assert_segments segments
+  end
+
+  def test_invalid_model_type
+    assert_raise TypeError do
+      Whisper::VAD::Context.new(Object.new)
+    end
+  end
+
+  def test_allocate
+    vad = Whisper::VAD::Context.allocate
+    assert_raise do
+      vad.detect(AUDIO, Whisper::VAD::Params.new)
+    end
+  end
+
+  private
+
+  def assert_segments(segments)
+    assert_instance_of Whisper::VAD::Segments, segments
+
+    i = 0
+    segments.each do |segment|
+      i += 1
+      assert_instance_of Whisper::VAD::Segment, segment
+    end
+    assert i > 0
+
+    segments.each_with_index do |segment, index|
+      assert_instance_of Integer, index
+    end
+
+    assert_instance_of Enumerator, segments.each
+
+    segment = segments.each.first
+    assert_instance_of Float, segment.start_time
+    assert_instance_of Float, segment.end_time
+
+    segment => {start_time:, end_time:}
+    assert_equal segment.start_time, start_time
+    assert_equal segment.end_time, end_time
+
+    assert_equal 4, segments.length
+  end
+
+  sub_test_case "from samples" do
+    def setup
+      super
+      @vad = Whisper::VAD::Context.new("silero-v6.2.0")
+      @samples = File.read(AUDIO, nil, 78).unpack("s<*").collect {|i| i.to_f / 2**15}
+    end
+
+    def test_segments_from_samples
+      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, @samples, @samples.length)
+      assert_segments segments
+    end
+
+    def test_segments_from_samples_without_length
+      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, @samples)
+      assert_segments segments
+    end
+
+    def test_segments_from_samples_enumerator
+      samples = @samples.each
+      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, samples, @samples.length)
+      assert_segments segments
+    end
+
+    def test_segments_from_samples_enumerator_without_length
+      samples = @samples.each
+      assert_raise ArgumentError do
+        @vad.segments_from_samples(Whisper::VAD::Params.new, samples)
+      end
+    end
+
+    def test_segments_from_samples_enumerator_with_too_large_length
+      samples = @samples.each.take(10).to_enum
+      assert_raise StopIteration do
+        @vad.segments_from_samples(Whisper::VAD::Params.new, samples, 11)
+      end
+    end
+
+    def test_segments_from_samples_with_memory_view
+      samples = JFKReader.new(AUDIO)
+      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, samples)
+      assert_segments segments
+    end
+  end
+end

data/test/test_vad_segment.rb

@@ -0,0 +1,19 @@
+require_relative "helper"
+
+class TestVADSegment < TestBase
+  def test_initialize
+    segment = Whisper::VAD::Segment.new
+
+    assert_raise do
+      segment.start_time
+    end
+
+    assert_raise do
+      segments.end_time
+    end
+
+    assert_raise do
+      segment => {start_time:, end_time:}
+    end
+  end
+end

data/test/test_vad_segments.rb

@@ -0,0 +1,16 @@
+require_relative "helper"
+
+class TestVADSegments < TestBase
+  def test_initialize
+    segments = Whisper::VAD::Segments.new
+
+    assert_raise do
+      segments.each do |segment|
+      end
+    end
+
+    assert_raise do
+      segments.length
+    end
+  end
+end

data/test/test_whisper.rb

@@ -1,6 +1,7 @@
 require_relative "helper"
 require "stringio"
 require "etc"
+require "pathname"
 
 # Exists to detect memory-related bug
 Whisper.log_set ->(level, buffer, user_data) {}, nil
@@ -20,6 +21,15 @@ class TestWhisper < TestBase
     }
   end
 
+  def test_whisper_pathname
+    @whisper = Whisper::Context.new("base.en")
+    params  = Whisper::Params.new
+
+    @whisper.transcribe(Pathname(AUDIO), params) {|text|
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, text)
+    }
+  end
+
   def test_transcribe_non_parallel
     @whisper = Whisper::Context.new("base.en")
     params  = Whisper::Params.new
@@ -149,6 +159,13 @@ class TestWhisper < TestBase
     $stderr = stderr
   end
 
+  def test_access_attribute_without_initialization
+    whisper = Whisper::Context.allocate
+    assert_raise do
+      whisper.model_type
+    end
+  end
+
   sub_test_case "full" do
     def setup
       super
@@ -200,6 +217,16 @@ class TestWhisper < TestBase
       assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)
     end
 
+    def test_full_with_memroy_view_gc
+      samples = JFKReader.new(AUDIO)
+      @whisper.full(@params, samples)
+      GC.start
+      require "fiddle"
+      Fiddle::MemoryView.export samples do |view|
+        assert_equal 176000, view.to_s.unpack("#{view.format}*").length
+      end
+    end
+
     def test_full_parallel
       nprocessors = 2
       @whisper.full_parallel(@params, @samples, @samples.length, nprocessors)

data/whispercpp.gemspec

@@ -3,7 +3,7 @@ require_relative "extsources"
 Gem::Specification.new do |s|
   s.name    = "whispercpp"
   s.authors = ["Georgi Gerganov", "Todd A. Fisher"]
-  s.version = '1.3.4'
+  s.version = '1.3.6'
   s.description = %q{High-performance inference of OpenAI's Whisper automatic speech recognition (ASR) model via Ruby}
   s.email   = 'todd.fisher@gmail.com'
   s.extra_rdoc_files = ['LICENSE', 'README.md']