whispercpp 1.3.5 → 1.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90c3f74c4f4c20ba9f9276f16fad4efa3458cab361f1dcdceb9ef436a3845ddc
-  data.tar.gz: 9001366168e697842700adc27305e0771c97e5ba3533c567c3cc6363004b2332
+  metadata.gz: 3fc84f88727ae65318081b59b6d94592baa0ec89808b3257788ff6e20a28a8fa
+  data.tar.gz: 4153ad098f2cf85fb021506982ab722b17886a33781d946878f9c65e4651cbff
 SHA512:
-  metadata.gz: d57baa8f961809c0323c83df79b5373cc2aefa38b5d21e3c538fea07ac6e74b95cfa86565bd64fd35ee385afb8baee32d9cb445fa07b7c51d4556ffcb0a0bda5
-  data.tar.gz: 6f1418ae638dc69fbff3d2accbb7652a360f6d21d34d003df7953054773bd7056add1051a302deb59649a6674c89fb22a419809ba09f166458f4da396945dd5f
+  metadata.gz: 934e01e49aea5a249e53cfdffa493805ac52160158fe05c1a9bf1d1063953fc8e1066dc75431a2cac900323e9c301ea9cea232072c8f30aabe38eff197d00021
+  data.tar.gz: d3fc7777152d984d2ca658e20706187c6a0546d428a18a068fe9c8fc0224699d4b7937062bf1b84fcf879e75a6aa9378e9a9e6e00f5daf59630f3f1b393f56b7

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023-2024 The ggml authors
+Copyright (c) 2023-2026 The ggml authors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -247,6 +247,58 @@ whisper.transcribe("path/to/audio.wav", params)
 
 ```
 
+### Tokens ###
+
+Each segment has tokens.
+
+To enable token timestamps, you need to set `Whisper::Params#token_timestamps = true`. Then, retrieve tokens from segments using `Whisper::Segment#each_token`.
+
+```ruby
+whisper = Whisper::Context.new("base.en")
+params = Whisper::Params.new(token_timestamps: true)
+whisper
+  .transcribe("path/to/audio.wav", params)
+  .each_segment do |segment|
+    segment.each_token do |token|
+      token => {start_time:, end_time:, text:, probability:}
+      st = "%05.2fs" % (start_time / 1000.0)
+      et = "%05.2fs" % (end_time / 1000.0)
+      prob = "%.1f%%" % (probability * 100)
+      puts "[#{st} --> #{et}] #{text} (#{prob})"
+    end
+  end
+```
+
+```
+[00.00s --> 00.00s] [_BEG_] (84.2%)
+[00.32s --> 00.37s]  And (71.2%)
+[00.37s --> 00.53s]  so (98.5%)
+[00.69s --> 00.85s]  my (70.7%)
+[00.85s --> 01.59s]  fellow (99.5%)
+[01.59s --> 02.10s]  Americans (90.1%)
+[02.85s --> 03.30s] , (28.4%)
+[03.30s --> 04.14s]  ask (79.8%)
+[04.14s --> 04.28s]  not (78.9%)
+[05.03s --> 05.35s]  what (93.3%)
+[05.41s --> 05.74s]  your (98.8%)
+[05.74s --> 06.41s]  country (99.6%)
+[06.41s --> 06.74s]  can (97.7%)
+[06.74s --> 06.92s]  do (99.0%)
+[07.00s --> 07.00s]  for (95.8%)
+[07.01s --> 07.52s]  you (98.5%)
+[07.81s --> 08.05s] , (49.3%)
+[08.19s --> 08.37s]  ask (65.6%)
+[08.37s --> 08.75s]  what (98.8%)
+[08.91s --> 09.04s]  you (98.2%)
+[09.04s --> 09.32s]  can (96.9%)
+[09.32s --> 09.38s]  do (90.3%)
+[09.44s --> 09.76s]  for (91.8%)
+[09.76s --> 09.99s]  your (98.2%)
+[10.02s --> 10.36s]  country (99.6%)
+[10.51s --> 10.99s] . (87.0%)
+[11.00s --> 11.00s] [_TT_550] (7.6%)
+```
+
 ### Models ###
 
 You can see model information:
@@ -323,7 +375,38 @@ whisper
   end
 ```
 
-The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView.
+
+If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+
+```ruby
+require "torchaudio"
+require "arrow-numo-narray"
+require "whisper"
+
+waveform, sample_rate = TorchAudio.load("test/fixtures/jfk.wav")
+# Convert Torch::Tensor to Arrow::Array via Numo::NArray
+samples = waveform.squeeze.numo.to_arrow.to_arrow_array
+
+whisper = Whisper::Context.new("base")
+whisper
+  # Arrow::Array exports MemoryView
+  .full(Whisper::Params.new, samples)
+```
+
+Custom context params
+---------------------
+
+You can use customize `Whisper::Context`'s behavior using `Whisper::Context::Params`.
+
+```ruby
+context_params = Whisper::Context::Params.new(
+  use_gpu: false,
+  flash_attn: false,
+  # etc
+)
+whisper = Whisper::Context.new("base", context_params)
+```
 
 Using VAD separately from ASR
 -----------------------------
@@ -334,13 +417,27 @@ VAD feature itself is useful. You can use it separately from ASR:
 vad = Whisper::VAD::Context.new("silero-v6.2.0")
 vad
   .detect("path/to/audio.wav", Whisper::VAD::Params.new)
-  .each_with_index do |segment, index|
+  .each.with_index do |segment, index|
     segment => {start_time: st, end_time: ed} # `Segment` responds to `#deconstruct_keys`
 
     puts "[%{nth}: %{st} --> %{ed}]" % {nth: index + 1, st:, ed:}
   end
 ```
 
+You may also low level API `Whisper::VAD::Context#segments_from_samples` as such `Whisper::Context#full`:
+
+```ruby
+# Ruby Array
+reader = WaveFile::Reader.new("path/to/audio.wav", WaveFile::Format.new(:mono, :float, 16000))
+samples = reader.enum_for(:each_buffer).map(&:samples).flatten
+
+# Or, object which exports MemoryView
+waveform, sample_rate = TorchAudio.load("test/fixtures/jfk.wav")
+samples = waveform.squeeze.numo.to_arrow.to_arrow_array
+
+segments = vad.segments_from_samples(Whisper::VAD::Params.new, samples)
+```
+
 Development
 -----------
 

data/ext/extconf.rb

@@ -7,6 +7,7 @@ options = Options.new(cmake).to_s
 have_library("gomp") rescue nil
 libs = Dependencies.new(cmake, options).to_s
 
+$CFLAGS << " -O3 -march=native"
 $INCFLAGS << " -Isources/include -Isources/ggml/include -Isources/examples"
 $LOCAL_LIBS << " #{libs}"
 $cleanfiles << " build #{libs}"

data/ext/ruby_whisper.c

@@ -1,5 +1,3 @@
-#include <ruby.h>
-#include <ruby/memory_view.h>
 #include "ruby_whisper.h"
 
 VALUE mWhisper;
@@ -35,7 +33,8 @@ static bool is_log_callback_finalized = false;
 // High level API
 extern VALUE ruby_whisper_segment_allocate(VALUE klass);
 
-extern void init_ruby_whisper_context(VALUE *mWhisper);
+extern VALUE init_ruby_whisper_context(VALUE *mWhisper);
+extern void init_ruby_whisper_context_params(VALUE *cContext);
 extern void init_ruby_whisper_params(VALUE *mWhisper);
 extern void init_ruby_whisper_error(VALUE *mWhisper);
 extern void init_ruby_whisper_segment(VALUE *mWhisper);
@@ -164,6 +163,22 @@ void Init_whisper() {
   rb_define_const(mWhisper, "LOG_LEVEL_DEBUG", INT2NUM(GGML_LOG_LEVEL_DEBUG));
   rb_define_const(mWhisper, "LOG_LEVEL_CONT", INT2NUM(GGML_LOG_LEVEL_CONT));
 
+  rb_define_const(mWhisper, "AHEADS_NONE", INT2NUM(WHISPER_AHEADS_NONE));
+  rb_define_const(mWhisper, "AHEADS_N_TOP_MOST", INT2NUM(WHISPER_AHEADS_N_TOP_MOST));
+  rb_define_const(mWhisper, "AHEADS_CUSTOM", INT2NUM(WHISPER_AHEADS_CUSTOM));
+  rb_define_const(mWhisper, "AHEADS_TINY_EN", INT2NUM(WHISPER_AHEADS_TINY_EN));
+  rb_define_const(mWhisper, "AHEADS_TINY", INT2NUM(WHISPER_AHEADS_TINY));
+  rb_define_const(mWhisper, "AHEADS_BASE_EN", INT2NUM(WHISPER_AHEADS_BASE_EN));
+  rb_define_const(mWhisper, "AHEADS_BASE", INT2NUM(WHISPER_AHEADS_BASE));
+  rb_define_const(mWhisper, "AHEADS_SMALL_EN", INT2NUM(WHISPER_AHEADS_SMALL_EN));
+  rb_define_const(mWhisper, "AHEADS_SMALL", INT2NUM(WHISPER_AHEADS_SMALL));
+  rb_define_const(mWhisper, "AHEADS_MEDIUM_EN", INT2NUM(WHISPER_AHEADS_MEDIUM_EN));
+  rb_define_const(mWhisper, "AHEADS_MEDIUM", INT2NUM(WHISPER_AHEADS_MEDIUM));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V1", INT2NUM(WHISPER_AHEADS_LARGE_V1));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V2", INT2NUM(WHISPER_AHEADS_LARGE_V2));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V3", INT2NUM(WHISPER_AHEADS_LARGE_V3));
+  rb_define_const(mWhisper, "AHEADS_LARGE_V3_TURBO", INT2NUM(WHISPER_AHEADS_LARGE_V3_TURBO));
+
   rb_define_singleton_method(mWhisper, "lang_max_id", ruby_whisper_s_lang_max_id, 0);
   rb_define_singleton_method(mWhisper, "lang_id", ruby_whisper_s_lang_id, 1);
   rb_define_singleton_method(mWhisper, "lang_str", ruby_whisper_s_lang_str, 1);
@@ -172,7 +187,8 @@ void Init_whisper() {
   rb_define_singleton_method(mWhisper, "log_set", ruby_whisper_s_log_set, 2);
   rb_define_private_method(rb_singleton_class(mWhisper), "finalize_log_callback", ruby_whisper_s_finalize_log_callback, 1);
 
-  init_ruby_whisper_context(&mWhisper);
+  cContext = init_ruby_whisper_context(&mWhisper);
+  init_ruby_whisper_context_params(&cContext);
   init_ruby_whisper_params(&mWhisper);
   init_ruby_whisper_error(&mWhisper);
   init_ruby_whisper_segment(&mWhisper);

data/ext/ruby_whisper.h

@@ -1,6 +1,8 @@
 #ifndef RUBY_WHISPER_H
 #define RUBY_WHISPER_H
 
+#include <ruby.h>
+#include <ruby/memory_view.h>
 #include "whisper.h"
 
 typedef struct {
@@ -14,6 +16,10 @@ typedef struct {
   struct whisper_context *context;
 } ruby_whisper;
 
+typedef struct ruby_whisper_context_params {
+  struct whisper_context_params params;
+} ruby_whisper_context_params;
+
 typedef struct {
   struct whisper_full_params params;
   bool diarize;
@@ -35,7 +41,7 @@ typedef struct {
 
 typedef struct {
   whisper_token_data *token_data;
-  const char *text;
+  VALUE text;
 } ruby_whisper_token;
 
 typedef struct {
@@ -55,6 +61,13 @@ typedef struct {
   struct whisper_vad_context *context;
 } ruby_whisper_vad_context;
 
+typedef struct parsed_samples_t {
+  float *samples;
+  int n_samples;
+  rb_memory_view_t memview;
+  bool memview_exported;
+} parsed_samples_t;
+
 #define GetContext(obj, rw) do { \
   TypedData_Get_Struct((obj), ruby_whisper, &ruby_whisper_type, (rw)); \
   if ((rw)->context == NULL) { \
@@ -62,13 +75,28 @@ typedef struct {
   } \
 } while (0)
 
-#define GetToken(obj, rwt) do {                                             \
+#define GetContextParams(obj, rwcp) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_context_params, &ruby_whisper_context_params_type, (rwcp)); \
+} while (0)
+
+#define GetToken(obj, rwt) do { \
   TypedData_Get_Struct((obj), ruby_whisper_token, &ruby_whisper_token_type, (rwt)); \
   if ((rwt)->token_data == NULL) { \
     rb_raise(rb_eRuntimeError, "Not initialized"); \
   } \
 } while (0)
 
+#define GetVADContext(obj, rwvc) do { \
+    TypedData_Get_Struct((obj), ruby_whisper_vad_context, &ruby_whisper_vad_context_type, (rwvc)); \
+    if ((rwvc)->context == NULL) { \
+      rb_raise(rb_eRuntimeError, "Not initialized"); \
+    } \
+} while (0)
+
+#define GetVADParams(obj, rwvp) do { \
+  TypedData_Get_Struct((obj), ruby_whisper_vad_params, &ruby_whisper_vad_params_type, (rwvp)); \
+} while (0)
+
 #define GetVADSegments(obj, rwvss) do { \
   TypedData_Get_Struct((obj), ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, (rwvss)); \
   if ((rwvss)->segments == NULL) { \