whispercpp 1.3.4 → 1.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d790b742ddf3f6071ba7cc41ea9a16d2daa1363d4fee9c984ad2609fb19ed9fc
-  data.tar.gz: 97b213ff5bc5cee1a2880d0e69d4ba9b89da309f003957a66f89ea7bdbfa1ee7
+  metadata.gz: 3fc84f88727ae65318081b59b6d94592baa0ec89808b3257788ff6e20a28a8fa
+  data.tar.gz: 4153ad098f2cf85fb021506982ab722b17886a33781d946878f9c65e4651cbff
 SHA512:
-  metadata.gz: e1ea6c6082532e90b26e17041bd0649f91e32fc70cb489bff0c9d83e4d92431e8d36a54d47904070f1d799e9df9d2fe7dc5f4cc668bf84c6339b7edf4c2b4d45
-  data.tar.gz: e2007190f93a5f6cb21d8014e07c650aa0e835286eb13131d268d02e563c519c23e1ccce60c90f7da5f921fb4ed1eaec143239627015fb6c40e02188e5eaf34c
+  metadata.gz: 934e01e49aea5a249e53cfdffa493805ac52160158fe05c1a9bf1d1063953fc8e1066dc75431a2cac900323e9c301ea9cea232072c8f30aabe38eff197d00021
+  data.tar.gz: d3fc7777152d984d2ca658e20706187c6a0546d428a18a068fe9c8fc0224699d4b7937062bf1b84fcf879e75a6aa9378e9a9e6e00f5daf59630f3f1b393f56b7

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023-2024 The ggml authors
+Copyright (c) 2023-2026 The ggml authors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -5,43 +5,6 @@ whispercpp
 
 Ruby bindings for [whisper.cpp][], an interface of automatic speech recognition model.
 
-Installation
-------------
-
-Install the gem and add to the application's Gemfile by executing:
-
-    $ bundle add whispercpp
-
-If bundler is not being used to manage dependencies, install the gem by executing:
-
-    $ gem install whispercpp
-
-You can pass build options for whisper.cpp, for instance:
-
-    $ bundle config build.whispercpp --enable-ggml-cuda
-
-or,
-
-    $ gem install whispercpp -- --enable-ggml-cuda
-
-See whisper.cpp's [README](https://github.com/ggml-org/whisper.cpp/blob/master/README.md) for available options. You need convert options present the README to Ruby-style options, for example:
-
-Boolean options:
-
-* `-DGGML_BLAS=1` -> `--enable-ggml-blas`
-* `-DWHISER_COREML=OFF` -> `--disable-whisper-coreml`
-
-Argument options:
-
-* `-DGGML_CUDA_COMPRESSION_MODE=size` -> `--ggml-cuda-compression-mode=size`
-
-Combination:
-
-* `-DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="86"` -> `--enable-ggml-cuda --cmake_cuda-architectures="86"`
-
-For boolean options like `GGML_CUDA`, the README says `-DGGML_CUDA=1`. You need strip `-D`, prepend `--enable-` for `1` or `ON` (`--disable-` for `0` or `OFF`) and make it kebab-case: `--enable-ggml-cuda`.  
-For options which require arguments like `CMAKE_CUDA_ARCHITECTURES`, the README says `-DCMAKE_CUDA_ARCHITECTURES="86"`. You need strip `-D`, prepend `--`, make it kebab-case, append `=` and append argument: `--cmake-cuda-architectures="86"`.
-
 Usage
 -----
 
@@ -57,7 +20,8 @@ params = Whisper::Params.new(
   max_text_tokens: 300,
   translate: true,
   print_timestamps: false,
-  initial_prompt: "Initial prompt here."
+  initial_prompt: "Initial prompt here.",
+  carry_initial_prompt: true
 )
 
 whisper.transcribe("path/to/audio.wav", params) do |whole_text|
@@ -118,7 +82,8 @@ Or, you can download model files:
 ```ruby
 whisper = Whisper::Context.new("https://example.net/uri/of/your/model.bin")
 # Or
-whisper = Whisper::Context.new(URI("https://example.net/uri/of/your/model.bin"))
+uri = URI("https://example.net/uri/of/your/model.bin")
+whisper = Whisper::Context.new(uri)
 ```
 
 See [models][] page for details.
@@ -134,20 +99,20 @@ Support for Voice Activity Detection (VAD) can be enabled by setting `Whisper::P
 ```ruby
 Whisper::Params.new(
   vad: true,
-  vad_model_path: "silero-v5.1.2",
+  vad_model_path: "silero-v6.2.0",
   # other arguments...
 )
 ```
 
-When you pass the model name (`"silero-v5.1.2"`) or URI (`https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v5.1.2.bin`), it will be downloaded automatically.
-Currently, "silero-v5.1.2" is registered as pre-converted model like ASR models. You also specify file path or URI of model.
+When you pass the model name (`"silero-v6.2.0"`) or URI (`https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin`), it will be downloaded automatically.
+Currently, "silero-v6.2.0" is registered as pre-converted model like ASR models. You also specify file path or URI of model.
 
 If you need configure VAD behavior, pass params for that:
 
 ```ruby
 Whisper::Params.new(
   vad: true,
-  vad_model_path: "silero-v5.1.2",
+  vad_model_path: "silero-v6.2.0",
   vad_params: Whisper::VAD::Params.new(
     threshold: 1.0, # defaults to 0.5
     min_speech_duration_ms: 500, # defaults to 250
@@ -187,6 +152,42 @@ WEBVTT
 
 You may call `#to_srt`, too
 
+Installation
+------------
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add whispercpp
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install whispercpp
+
+You can pass build options for whisper.cpp, for instance:
+
+    $ bundle config build.whispercpp --enable-ggml-cuda
+
+or,
+
+    $ gem install whispercpp -- --enable-ggml-cuda
+
+See whisper.cpp's [README](https://github.com/ggml-org/whisper.cpp/blob/master/README.md) for available options. You need convert options present in the README to Ruby-style options, for example:
+
+Boolean options:
+
+* `-DGGML_BLAS=1` -> `--enable-ggml-blas`
+* `-DWHISER_COREML=OFF` -> `--disable-whisper-coreml`
+
+Argument options:
+
+* `-DGGML_CUDA_COMPRESSION_MODE=size` -> `--ggml-cuda-compression-mode=size`
+
+Combination:
+
+* `-DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="86"` -> `--enable-ggml-cuda --cmake_cuda-architectures="86"`
+
+For boolean options like `GGML_CUDA`, the README says `-DGGML_CUDA=1`. You need strip `-D`, prepend `--enable-` for `1` or `ON` (`--disable-` for `0` or `OFF`) and make it kebab-case: `--enable-ggml-cuda`.  
+For options which require arguments like `CMAKE_CUDA_ARCHITECTURES`, the README says `-DCMAKE_CUDA_ARCHITECTURES="86"`. You need strip `-D`, prepend `--`, make it kebab-case, append `=` and append argument: `--cmake-cuda-architectures="86"`.
 
 API
 ---
@@ -246,6 +247,58 @@ whisper.transcribe("path/to/audio.wav", params)
 
 ```
 
+### Tokens ###
+
+Each segment has tokens.
+
+To enable token timestamps, you need to set `Whisper::Params#token_timestamps = true`. Then, retrieve tokens from segments using `Whisper::Segment#each_token`.
+
+```ruby
+whisper = Whisper::Context.new("base.en")
+params = Whisper::Params.new(token_timestamps: true)
+whisper
+  .transcribe("path/to/audio.wav", params)
+  .each_segment do |segment|
+    segment.each_token do |token|
+      token => {start_time:, end_time:, text:, probability:}
+      st = "%05.2fs" % (start_time / 1000.0)
+      et = "%05.2fs" % (end_time / 1000.0)
+      prob = "%.1f%%" % (probability * 100)
+      puts "[#{st} --> #{et}] #{text} (#{prob})"
+    end
+  end
+```
+
+```
+[00.00s --> 00.00s] [_BEG_] (84.2%)
+[00.32s --> 00.37s]  And (71.2%)
+[00.37s --> 00.53s]  so (98.5%)
+[00.69s --> 00.85s]  my (70.7%)
+[00.85s --> 01.59s]  fellow (99.5%)
+[01.59s --> 02.10s]  Americans (90.1%)
+[02.85s --> 03.30s] , (28.4%)
+[03.30s --> 04.14s]  ask (79.8%)
+[04.14s --> 04.28s]  not (78.9%)
+[05.03s --> 05.35s]  what (93.3%)
+[05.41s --> 05.74s]  your (98.8%)
+[05.74s --> 06.41s]  country (99.6%)
+[06.41s --> 06.74s]  can (97.7%)
+[06.74s --> 06.92s]  do (99.0%)
+[07.00s --> 07.00s]  for (95.8%)
+[07.01s --> 07.52s]  you (98.5%)
+[07.81s --> 08.05s] , (49.3%)
+[08.19s --> 08.37s]  ask (65.6%)
+[08.37s --> 08.75s]  what (98.8%)
+[08.91s --> 09.04s]  you (98.2%)
+[09.04s --> 09.32s]  can (96.9%)
+[09.32s --> 09.38s]  do (90.3%)
+[09.44s --> 09.76s]  for (91.8%)
+[09.76s --> 09.99s]  your (98.2%)
+[10.02s --> 10.36s]  country (99.6%)
+[10.51s --> 10.99s] . (87.0%)
+[11.00s --> 11.00s] [_TT_550] (7.6%)
+```
+
 ### Models ###
 
 You can see model information:
@@ -322,7 +375,68 @@ whisper
   end
 ```
 
-The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView.
+
+If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+
+```ruby
+require "torchaudio"
+require "arrow-numo-narray"
+require "whisper"
+
+waveform, sample_rate = TorchAudio.load("test/fixtures/jfk.wav")
+# Convert Torch::Tensor to Arrow::Array via Numo::NArray
+samples = waveform.squeeze.numo.to_arrow.to_arrow_array
+
+whisper = Whisper::Context.new("base")
+whisper
+  # Arrow::Array exports MemoryView
+  .full(Whisper::Params.new, samples)
+```
+
+Custom context params
+---------------------
+
+You can use customize `Whisper::Context`'s behavior using `Whisper::Context::Params`.
+
+```ruby
+context_params = Whisper::Context::Params.new(
+  use_gpu: false,
+  flash_attn: false,
+  # etc
+)
+whisper = Whisper::Context.new("base", context_params)
+```
+
+Using VAD separately from ASR
+-----------------------------
+
+VAD feature itself is useful. You can use it separately from ASR:
+
+```ruby
+vad = Whisper::VAD::Context.new("silero-v6.2.0")
+vad
+  .detect("path/to/audio.wav", Whisper::VAD::Params.new)
+  .each.with_index do |segment, index|
+    segment => {start_time: st, end_time: ed} # `Segment` responds to `#deconstruct_keys`
+
+    puts "[%{nth}: %{st} --> %{ed}]" % {nth: index + 1, st:, ed:}
+  end
+```
+
+You may also low level API `Whisper::VAD::Context#segments_from_samples` as such `Whisper::Context#full`:
+
+```ruby
+# Ruby Array
+reader = WaveFile::Reader.new("path/to/audio.wav", WaveFile::Format.new(:mono, :float, 16000))
+samples = reader.enum_for(:each_buffer).map(&:samples).flatten
+
+# Or, object which exports MemoryView
+waveform, sample_rate = TorchAudio.load("test/fixtures/jfk.wav")
+samples = waveform.squeeze.numo.to_arrow.to_arrow_array
+
+segments = vad.segments_from_samples(Whisper::VAD::Params.new, samples)
+```
 
 Development
 -----------

data/ext/extconf.rb

@@ -3,10 +3,11 @@ require_relative "options"
 require_relative "dependencies"
 
 cmake = find_executable("cmake") || abort
-options = Options.new(cmake)
+options = Options.new(cmake).to_s
 have_library("gomp") rescue nil
-libs = Dependencies.new(cmake, options)
+libs = Dependencies.new(cmake, options).to_s
 
+$CFLAGS << " -O3 -march=native"
 $INCFLAGS << " -Isources/include -Isources/ggml/include -Isources/examples"
 $LOCAL_LIBS << " #{libs}"
 $cleanfiles << " build #{libs}"

data/ext/ruby_whisper.c

@@ -1,15 +1,17 @@
-#include <ruby.h>
-#include <ruby/memory_view.h>
 #include "ruby_whisper.h"
 
 VALUE mWhisper;
 VALUE mVAD;
 VALUE cContext;
 VALUE cParams;
+VALUE cVADContext;
 VALUE cVADParams;
+VALUE cVADSegments;
+VALUE cVADSegment;
 VALUE eError;
 
 VALUE cSegment;
+VALUE cToken;
 VALUE cModel;
 
 ID id_to_s;
@@ -31,12 +33,17 @@ static bool is_log_callback_finalized = false;
 // High level API
 extern VALUE ruby_whisper_segment_allocate(VALUE klass);
 
-extern void init_ruby_whisper_context(VALUE *mWhisper);
+extern VALUE init_ruby_whisper_context(VALUE *mWhisper);
+extern void init_ruby_whisper_context_params(VALUE *cContext);
 extern void init_ruby_whisper_params(VALUE *mWhisper);
 extern void init_ruby_whisper_error(VALUE *mWhisper);
-extern void init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cSegment);
+extern void init_ruby_whisper_segment(VALUE *mWhisper);
+extern void init_ruby_whisper_token(VALUE *mWhisper);
 extern void init_ruby_whisper_model(VALUE *mWhisper);
 extern void init_ruby_whisper_vad_params(VALUE *mVAD);
+extern void init_ruby_whisper_vad_context(VALUE *mVAD);
+extern void init_ruby_whisper_vad_segment(VALUE *mVAD);
+extern void init_ruby_whisper_vad_segments(VALUE *mVAD);
 extern void register_callbacks(ruby_whisper_params *rwp, VALUE *context);
 
 /*
@@ -156,6 +163,22 @@ void Init_whisper() {
   rb_define_const(mWhisper, "LOG_LEVEL_DEBUG", INT2NUM(GGML_LOG_LEVEL_DEBUG));
   rb_define_const(mWhisper, "LOG_LEVEL_CONT", INT2NUM(GGML_LOG_LEVEL_CONT));
 
+  rb_define_const(mWhisper, "AHEADS_NONE", INT2NUM(WHISPER_AHEADS_NONE));
+  rb_define_const(mWhisper, "AHEADS_N_TOP_MOST", INT2NUM(WHISPER_AHEADS_N_TOP_MOST));
+  rb_define_const(mWhisper, "AHEADS_CUSTOM", INT2NUM(WHISPER_AHEADS_CUSTOM));
+  rb_define_const(mWhisper, "AHEADS_TINY_EN", INT2NUM(WHISPER_AHEADS_TINY_EN));
+  rb_define_const(mWhisper, "AHEADS_TINY", INT2NUM(WHISPER_AHEADS_TINY));
+  rb_define_const(mWhisper, "AHEADS_BASE_EN", INT2NUM(WHISPER_AHEADS_BASE_EN));
+  rb_define_const(mWhisper, "AHEADS_BASE", INT2NUM(WHISPER_AHEADS_BASE));
+  rb_define_const(mWhisper, "AHEADS_SMALL_EN", INT2NUM(WHISPER_AHEADS_SMALL_EN));
+  rb_define_const(mWhisper, "AHEADS_SMALL", INT2NUM(WHISPER_AHEADS_SMALL));
+  rb_define_const(mWhisper, "AHEADS_MEDIUM_EN", INT2NUM(WHISPER_AHEADS_MEDIUM_EN));
+  rb_define_const(mWhisper, "AHEADS_MEDIUM", INT2NUM(WHISPER_AHEADS_MEDIUM));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V1", INT2NUM(WHISPER_AHEADS_LARGE_V1));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V2", INT2NUM(WHISPER_AHEADS_LARGE_V2));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V3", INT2NUM(WHISPER_AHEADS_LARGE_V3));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V3_TURBO", INT2NUM(WHISPER_AHEADS_LARGE_V3_TURBO));
+
   rb_define_singleton_method(mWhisper, "lang_max_id", ruby_whisper_s_lang_max_id, 0);
   rb_define_singleton_method(mWhisper, "lang_id", ruby_whisper_s_lang_id, 1);
   rb_define_singleton_method(mWhisper, "lang_str", ruby_whisper_s_lang_str, 1);
@@ -164,12 +187,17 @@ void Init_whisper() {
   rb_define_singleton_method(mWhisper, "log_set", ruby_whisper_s_log_set, 2);
   rb_define_private_method(rb_singleton_class(mWhisper), "finalize_log_callback", ruby_whisper_s_finalize_log_callback, 1);
 
-  init_ruby_whisper_context(&mWhisper);
+  cContext = init_ruby_whisper_context(&mWhisper);
+  init_ruby_whisper_context_params(&cContext);
   init_ruby_whisper_params(&mWhisper);
   init_ruby_whisper_error(&mWhisper);
-  init_ruby_whisper_segment(&mWhisper, &cContext);
+  init_ruby_whisper_segment(&mWhisper);
+  init_ruby_whisper_token(&mWhisper);
   init_ruby_whisper_model(&mWhisper);
   init_ruby_whisper_vad_params(&mVAD);
+  init_ruby_whisper_vad_segment(&mVAD);
+  init_ruby_whisper_vad_segments(&mVAD);
+  init_ruby_whisper_vad_context(&mVAD);
 
   rb_require("whisper/context");
   rb_require("whisper/segment");

data/ext/ruby_whisper.h

@@ -1,6 +1,8 @@
 #ifndef RUBY_WHISPER_H
 #define RUBY_WHISPER_H
 
+#include <ruby.h>
+#include <ruby/memory_view.h>
 #include "whisper.h"
 
 typedef struct {
@@ -14,6 +16,10 @@ typedef struct {
   struct whisper_context *context;
 } ruby_whisper;
 
+typedef struct ruby_whisper_context_params {
+  struct whisper_context_params params;
+} ruby_whisper_context_params;
+
 typedef struct {
   struct whisper_full_params params;
   bool diarize;
@@ -33,8 +39,69 @@ typedef struct {
   int index;
 } ruby_whisper_segment;
 
+typedef struct {
+  whisper_token_data *token_data;
+  VALUE text;
+} ruby_whisper_token;
+
 typedef struct {
   VALUE context;
 } ruby_whisper_model;
 
+typedef struct {
+  struct whisper_vad_segments *segments;
+} ruby_whisper_vad_segments;
+
+typedef struct {
+  VALUE segments;
+  int index;
+} ruby_whisper_vad_segment;
+
+typedef struct {
+  struct whisper_vad_context *context;
+} ruby_whisper_vad_context;
+
+typedef struct parsed_samples_t {
+  float *samples;
+  int n_samples;
+  rb_memory_view_t memview;
+  bool memview_exported;
+} parsed_samples_t;
+
+#define GetContext(obj, rw) do { \
+  TypedData_Get_Struct((obj), ruby_whisper, &ruby_whisper_type, (rw)); \
+  if ((rw)->context == NULL) { \
+    rb_raise(rb_eRuntimeError, "Not initialized"); \
+  } \
+} while (0)
+
+#define GetContextParams(obj, rwcp) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_context_params, &ruby_whisper_context_params_type, (rwcp)); \
+} while (0)
+
+#define GetToken(obj, rwt) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_token, &ruby_whisper_token_type, (rwt)); \
+  if ((rwt)->token_data == NULL) { \
+    rb_raise(rb_eRuntimeError, "Not initialized"); \
+  } \
+} while (0)
+
+#define GetVADContext(obj, rwvc) do { \
+    TypedData_Get_Struct((obj), ruby_whisper_vad_context, &ruby_whisper_vad_context_type, (rwvc)); \
+    if ((rwvc)->context == NULL) { \
+      rb_raise(rb_eRuntimeError, "Not initialized"); \
+    } \
+} while (0)
+
+#define GetVADParams(obj, rwvp) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_vad_params, &ruby_whisper_vad_params_type, (rwvp)); \
+} while (0)
+
+#define GetVADSegments(obj, rwvss) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, (rwvss)); \
+  if ((rwvss)->segments == NULL) { \
+    rb_raise(rb_eRuntimeError, "Not initialized"); \
+  } \
+} while (0)
+
 #endif