whispercpp 1.3.3 → 1.3.5

data/ext/ruby_whisper_params.c

@@ -26,7 +26,7 @@
   rb_define_method(cParams, #param_name, ruby_whisper_params_get_ ## param_name, 0); \
   rb_define_method(cParams, #param_name "=", ruby_whisper_params_set_ ## param_name, 1);
 
-#define RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT 35
+#define RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT 37
 
 extern VALUE cParams;
 extern VALUE cVADParams;
@@ -46,9 +46,11 @@ static ID id_print_special;
 static ID id_print_progress;
 static ID id_print_realtime;
 static ID id_print_timestamps;
+static ID id_carry_initial_prompt;
 static ID id_suppress_blank;
 static ID id_suppress_nst;
 static ID id_token_timestamps;
+static ID id_max_len;
 static ID id_split_on_word;
 static ID id_initial_prompt;
 static ID id_diarize;
@@ -426,6 +428,7 @@ ruby_whisper_params_set_print_realtime(VALUE self, VALUE value)
 }
 /*
  * If true, prints results from within whisper.cpp. (avoid it, use callback instead)
+ *
  * call-seq:
  *   print_realtime -> bool
  */
@@ -454,6 +457,26 @@ ruby_whisper_params_get_print_timestamps(VALUE self)
 {
   BOOL_PARAMS_GETTER(self, print_timestamps)
 }
+
+/*
+ *  call-seq:
+ *    carry_initial_prompt -> true or false
+ */
+static VALUE
+ruby_whisper_params_get_carry_initial_prompt(VALUE self)
+{
+  BOOL_PARAMS_GETTER(self, carry_initial_prompt)
+}
+
+/*
+ *  call-seq:
+ *    carry_initial_prompt = bool -> bool
+ */
+static VALUE
+ruby_whisper_params_set_carry_initial_prompt(VALUE self, VALUE value)
+{
+  BOOL_PARAMS_SETTER(self, carry_initial_prompt, value)
+}
 /*
  * call-seq:
  *   suppress_blank = force_suppress -> force_suppress
@@ -514,6 +537,33 @@ ruby_whisper_params_set_token_timestamps(VALUE self, VALUE value)
 {
   BOOL_PARAMS_SETTER(self, token_timestamps, value)
 }
+
+/*
+ * max segment length in characters.
+ *
+ * call-seq:
+ *   max_len -> Integer
+ */
+static VALUE
+ruby_whisper_params_get_max_len(VALUE self)
+{
+  ruby_whisper_params *rwp;
+  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);
+  return INT2NUM(rwp->params.max_len);
+}
+/*
+ * call-seq:
+ *   max_len = length -> length
+ */
+static VALUE
+ruby_whisper_params_set_max_len(VALUE self, VALUE value)
+{
+  ruby_whisper_params *rwp;
+  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);
+  rwp->params.max_len = NUM2INT(value);
+  return value;
+}
+
 /*
  * If true, split on word rather than on token (when used with max_len).
  *
@@ -1137,8 +1187,10 @@ ruby_whisper_params_initialize(int argc, VALUE *argv, VALUE self)
       SET_PARAM_IF_SAME(suppress_blank)
       SET_PARAM_IF_SAME(suppress_nst)
       SET_PARAM_IF_SAME(token_timestamps)
+      SET_PARAM_IF_SAME(max_len)
       SET_PARAM_IF_SAME(split_on_word)
       SET_PARAM_IF_SAME(initial_prompt)
+      SET_PARAM_IF_SAME(carry_initial_prompt)
       SET_PARAM_IF_SAME(offset)
       SET_PARAM_IF_SAME(duration)
       SET_PARAM_IF_SAME(max_text_tokens)
@@ -1271,30 +1323,32 @@ init_ruby_whisper_params(VALUE *mWhisper)
   DEFINE_PARAM(suppress_blank, 8)
   DEFINE_PARAM(suppress_nst, 9)
   DEFINE_PARAM(token_timestamps, 10)
-  DEFINE_PARAM(split_on_word, 11)
-  DEFINE_PARAM(initial_prompt, 12)
-  DEFINE_PARAM(diarize, 13)
-  DEFINE_PARAM(offset, 14)
-  DEFINE_PARAM(duration, 15)
-  DEFINE_PARAM(max_text_tokens, 16)
-  DEFINE_PARAM(temperature, 17)
-  DEFINE_PARAM(max_initial_ts, 18)
-  DEFINE_PARAM(length_penalty, 19)
-  DEFINE_PARAM(temperature_inc, 20)
-  DEFINE_PARAM(entropy_thold, 21)
-  DEFINE_PARAM(logprob_thold, 22)
-  DEFINE_PARAM(no_speech_thold, 23)
-  DEFINE_PARAM(new_segment_callback, 24)
-  DEFINE_PARAM(new_segment_callback_user_data, 25)
-  DEFINE_PARAM(progress_callback, 26)
-  DEFINE_PARAM(progress_callback_user_data, 27)
-  DEFINE_PARAM(encoder_begin_callback, 28)
-  DEFINE_PARAM(encoder_begin_callback_user_data, 29)
-  DEFINE_PARAM(abort_callback, 30)
-  DEFINE_PARAM(abort_callback_user_data, 31)
-  DEFINE_PARAM(vad, 32)
-  DEFINE_PARAM(vad_model_path, 33)
-  DEFINE_PARAM(vad_params, 34)
+  DEFINE_PARAM(max_len, 11)
+  DEFINE_PARAM(split_on_word, 12)
+  DEFINE_PARAM(initial_prompt, 13)
+  DEFINE_PARAM(carry_initial_prompt, 14)
+  DEFINE_PARAM(diarize, 15)
+  DEFINE_PARAM(offset, 16)
+  DEFINE_PARAM(duration, 17)
+  DEFINE_PARAM(max_text_tokens, 18)
+  DEFINE_PARAM(temperature, 19)
+  DEFINE_PARAM(max_initial_ts, 20)
+  DEFINE_PARAM(length_penalty, 21)
+  DEFINE_PARAM(temperature_inc, 22)
+  DEFINE_PARAM(entropy_thold, 23)
+  DEFINE_PARAM(logprob_thold, 24)
+  DEFINE_PARAM(no_speech_thold, 25)
+  DEFINE_PARAM(new_segment_callback, 26)
+  DEFINE_PARAM(new_segment_callback_user_data, 27)
+  DEFINE_PARAM(progress_callback, 28)
+  DEFINE_PARAM(progress_callback_user_data, 29)
+  DEFINE_PARAM(encoder_begin_callback, 30)
+  DEFINE_PARAM(encoder_begin_callback_user_data, 31)
+  DEFINE_PARAM(abort_callback, 32)
+  DEFINE_PARAM(abort_callback_user_data, 33)
+  DEFINE_PARAM(vad, 34)
+  DEFINE_PARAM(vad_model_path, 35)
+  DEFINE_PARAM(vad_params, 36)
 
   rb_define_method(cParams, "on_new_segment", ruby_whisper_params_on_new_segment, 0);
   rb_define_method(cParams, "on_progress", ruby_whisper_params_on_progress, 0);

data/ext/ruby_whisper_segment.c

@@ -1,19 +1,23 @@
 #include <ruby.h>
 #include "ruby_whisper.h"
 
-#define N_KEY_NAMES 5
+#define N_KEY_NAMES 6
 
+extern ID id___method__;
+extern ID id_to_enum;
 static VALUE sym_start_time;
 static VALUE sym_end_time;
 static VALUE sym_text;
 static VALUE sym_no_speech_prob;
 static VALUE sym_speaker_turn_next;
-static VALUE key_names;
+static VALUE sym_n_tokens;
 
 extern const rb_data_type_t ruby_whisper_type;
 
 extern VALUE cSegment;
 
+extern VALUE ruby_whisper_token_s_init(struct whisper_context *context, int i_segment, int index);
+
 static void
 rb_whisper_segment_mark(void *p)
 {
@@ -29,6 +33,9 @@ ruby_whisper_segment_memsize(const void *p)
   if (!rws) {
     return 0;
   }
+  if (rws->index) {
+    size += sizeof(rws->index);
+  }
   return size;
 }
 
@@ -69,7 +76,7 @@ ruby_whisper_segment_get_start_time(VALUE self)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
   const int64_t t0 = whisper_full_get_segment_t0(rw->context, rws->index);
   // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it
   return LONG2NUM(t0 * 10);
@@ -87,7 +94,7 @@ ruby_whisper_segment_get_end_time(VALUE self)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
   const int64_t t1 = whisper_full_get_segment_t1(rw->context, rws->index);
   // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it
   return LONG2NUM(t1 * 10);
@@ -105,7 +112,7 @@ ruby_whisper_segment_get_speaker_turn_next(VALUE self)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
   return whisper_full_get_segment_speaker_turn_next(rw->context, rws->index) ? Qtrue : Qfalse;
 }
 
@@ -119,7 +126,7 @@ ruby_whisper_segment_get_text(VALUE self)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
   const char * text = whisper_full_get_segment_text(rw->context, rws->index);
   return rb_str_new2(text);
 }
@@ -134,15 +141,67 @@ ruby_whisper_segment_get_no_speech_prob(VALUE self)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
   return DBL2NUM(whisper_full_get_segment_no_speech_prob(rw->context, rws->index));
 }
 
+/*
+ * Get number of tokens in the segment
+ *
+ * call-seq:
+ *   n_tokens -> Integer
+ */
+static VALUE
+ruby_whisper_segment_get_n_tokens(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
+  ruby_whisper *rw;
+  GetContext(rws->context, rw);
+  return INT2NUM(whisper_full_n_tokens(rw->context, rws->index));
+}
+
+/*
+ * Yields each Whisper::Token:
+ *
+ *   whisper.each_segment.first.each_token do |token|
+ *     p token
+ *   end
+ *
+ * Returns an Enumerator if no block is given:
+ *
+ *   whisper.each_segment.first.each_token.to_a # => [#<Whisper::Token>, ...]
+ *
+ * call-seq:
+ *   each_token {|token| ... }
+ *   each_token -> Enumerator
+ */
+static VALUE
+ruby_whisper_segment_each_token(VALUE self)
+{
+  if (!rb_block_given_p()) {
+    const VALUE method_name = rb_funcall(self, id___method__, 0);
+    return rb_funcall(self, id_to_enum, 1, method_name);
+  }
+
+  ruby_whisper_segment *rws;
+  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
+  ruby_whisper *rw;
+  GetContext(rws->context, rw);
+
+  const int n_tokens = whisper_full_n_tokens(rw->context, rws->index);
+  for (int i = 0; i < n_tokens; ++i) {
+    rb_yield(ruby_whisper_token_s_init(rw->context, rws->index, i));
+  }
+
+  return self;
+}
+
 /*
  * call-seq:
  *   deconstruct_keys(keys) -> hash
  *
- *  Possible keys: :start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next
+ *  Possible keys: :start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next, :n_tokens
  *
  *   whisper.each_segment do |segment|
  *     segment => {start_time:, end_time:, text:, no_speech_prob:, speaker_turn_next:}
@@ -156,12 +215,19 @@ ruby_whisper_segment_deconstruct_keys(VALUE self, VALUE keys)
   ruby_whisper_segment *rws;
   TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);
   ruby_whisper *rw;
-  TypedData_Get_Struct(rws->context, ruby_whisper, &ruby_whisper_type, rw);
+  GetContext(rws->context, rw);
 
   VALUE hash = rb_hash_new();
   long n_keys;
   if (NIL_P(keys)) {
-    keys = key_names;
+    keys = rb_ary_new3(
+      N_KEY_NAMES,
+      sym_start_time,
+      sym_end_time,
+      sym_text,
+      sym_no_speech_prob,
+      sym_speaker_turn_next
+    );
     n_keys = N_KEY_NAMES;
   } else {
     n_keys = RARRAY_LEN(keys);
@@ -186,13 +252,16 @@ ruby_whisper_segment_deconstruct_keys(VALUE self, VALUE keys)
     if (key == sym_speaker_turn_next) {
       rb_hash_aset(hash, key, ruby_whisper_segment_get_speaker_turn_next(self));
     }
+    if (key == sym_n_tokens) {
+      rb_hash_aset(hash, key, ruby_whisper_segment_get_n_tokens(self));
+    }
   }
 
   return hash;
 }
 
 void
-init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cContext)
+init_ruby_whisper_segment(VALUE *mWhisper)
 {
   cSegment  = rb_define_class_under(*mWhisper, "Segment", rb_cObject);
 
@@ -201,14 +270,7 @@ init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cContext)
   sym_text = ID2SYM(rb_intern("text"));
   sym_no_speech_prob = ID2SYM(rb_intern("no_speech_prob"));
   sym_speaker_turn_next = ID2SYM(rb_intern("speaker_turn_next"));
-  key_names = rb_ary_new3(
-    N_KEY_NAMES,
-    sym_start_time,
-    sym_end_time,
-    sym_text,
-    sym_no_speech_prob,
-    sym_speaker_turn_next
-  );
+  sym_n_tokens = ID2SYM(rb_intern("n_tokens"));
 
   rb_define_alloc_func(cSegment, ruby_whisper_segment_allocate);
   rb_define_method(cSegment, "start_time", ruby_whisper_segment_get_start_time, 0);
@@ -216,5 +278,8 @@ init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cContext)
   rb_define_method(cSegment, "speaker_turn_next?", ruby_whisper_segment_get_speaker_turn_next, 0);
   rb_define_method(cSegment, "text", ruby_whisper_segment_get_text, 0);
   rb_define_method(cSegment, "no_speech_prob", ruby_whisper_segment_get_no_speech_prob, 0);
+  rb_define_method(cSegment, "n_tokens", ruby_whisper_segment_get_n_tokens, 0);
+  rb_define_method(cSegment, "each_token", ruby_whisper_segment_each_token, 0);
   rb_define_method(cSegment, "deconstruct_keys", ruby_whisper_segment_deconstruct_keys, 1);
 }
+#undef N_KEY_NAMES

data/ext/ruby_whisper_token.c

@@ -0,0 +1,351 @@
+#include <ruby.h>
+#include "ruby_whisper.h"
+
+#define N_KEY_NAMES 11
+
+extern VALUE cToken;
+extern const rb_data_type_t ruby_whisper_type;
+
+static VALUE sym_id;
+static VALUE sym_tid;
+static VALUE sym_probability;
+static VALUE sym_log_probability;
+static VALUE sym_pt;
+static VALUE sym_ptsum;
+static VALUE sym_t_dtw;
+static VALUE sym_voice_length;
+static VALUE sym_start_time;
+static VALUE sym_end_time;
+static VALUE sym_text;
+
+static size_t
+ruby_whisper_token_memsize(const void *p)
+{
+  const ruby_whisper_token *rwt = (const ruby_whisper_token *)p;
+  if (!rwt) {
+    return 0;
+  }
+  return sizeof(rwt);
+}
+
+static const rb_data_type_t ruby_whisper_token_type = {
+  "ruby_whisper_token",
+  {0, RUBY_DEFAULT_FREE, ruby_whisper_token_memsize,},
+  0, 0,
+  0
+};
+
+static VALUE
+ruby_whisper_token_allocate(VALUE klass)
+{
+  ruby_whisper_token *rwt;
+  VALUE token = TypedData_Make_Struct(klass, ruby_whisper_token, &ruby_whisper_token_type, rwt);
+  rwt->token_data = NULL;
+  rwt->text = NULL;
+  return token;
+}
+
+VALUE
+ruby_whisper_token_s_init(struct whisper_context *context, int i_segment, int i_token)
+{
+  whisper_token_data token_data = whisper_full_get_token_data(context, i_segment, i_token);
+  const VALUE token = ruby_whisper_token_allocate(cToken);
+  ruby_whisper_token *rwt;
+  TypedData_Get_Struct(token, ruby_whisper_token, &ruby_whisper_token_type, rwt);
+  rwt->token_data = &token_data;
+  rwt->text = whisper_full_get_token_text(context, i_segment, i_token);
+  return token;
+}
+
+/*
+ * Token ID.
+ *
+ * call-seq:
+ *   id -> Integer
+ */
+static VALUE
+ruby_whisper_token_get_id(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return INT2NUM(rwt->token_data->id);
+}
+
+/*
+ * Forced timestamp token ID.
+ *
+ * call-seq:
+ *   tid -> Integer
+ */
+static VALUE
+ruby_whisper_token_get_tid(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return INT2NUM(rwt->token_data->tid);
+}
+
+/*
+ * Probability of the token.
+ *
+ * call-seq:
+ *   probability -> Float
+ */
+static VALUE
+ruby_whisper_token_get_p(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return DBL2NUM(rwt->token_data->p);
+}
+
+/*
+ * Log probability of the token.
+ *
+ * call-seq:
+ *   log_probability -> Float
+ */
+static VALUE
+ruby_whisper_token_get_plog(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return DBL2NUM(rwt->token_data->plog);
+}
+
+/*
+ * Probability of the timestamp token.
+ *
+ * call-seq:
+ *   pt -> Float
+ */
+static VALUE
+ruby_whisper_token_get_pt(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return DBL2NUM(rwt->token_data->pt);
+}
+
+/*
+ * Sum of probability of all timestamp tokens.
+ *
+ * call-seq:
+ *   ptsum -> Float
+ */
+static VALUE
+ruby_whisper_token_get_ptsum(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return DBL2NUM(rwt->token_data->ptsum);
+}
+
+/*
+ * [EXPERIMENTAL] Token-level timestamps with DTW
+ *
+ * Do not use if you haven't computed token-level timestamps with dtw.
+ * Roughly corresponds to the moment in audio in which the token was output.
+ *
+ * call-seq:
+ *   t_dtw -> Integer
+ */
+static VALUE
+ruby_whisper_token_get_t_dtw(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return LONG2NUM(rwt->token_data->t_dtw);
+}
+
+/*
+ * Voice length of the token.
+ *
+ * call-seq:
+ *   voice_length -> Float
+ */
+static VALUE
+ruby_whisper_token_get_vlen(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return DBL2NUM(rwt->token_data->vlen);
+}
+
+/*
+ * Get the token text of the token.
+ *
+ * call-seq:
+ *   text -> String
+ */
+static VALUE
+ruby_whisper_token_get_text(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return rb_str_new2(rwt->text);
+}
+
+
+/*
+ * Start time of the token.
+ *
+ * Token-level timestamp data.
+ * Do not use if you haven't computed token-level timestamps.
+ *
+ * call-seq:
+ *   start_time -> Integer
+ */
+static VALUE
+ruby_whisper_token_get_start_time(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return LONG2NUM(rwt->token_data->t0 * 10);
+}
+
+/*
+ * End time of the token.
+ *
+ * Token-level timestamp data.
+ * Do not use if you haven't computed token-level timestamps.
+ *
+ * call-seq:
+ *   end_time -> Integer
+ */
+static VALUE
+ruby_whisper_token_get_end_time(VALUE self)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  return LONG2NUM(rwt->token_data->t1 * 10);
+}
+
+/*
+ * call-seq:
+ *   deconstruct_keys(keys) -> hash
+ *
+ *  Possible keys: :id, :tid, :probability, :log_probability, :pt, :ptsum,
+ *                 :t_dtw, :voice_length, :start_time, :end_time, :text
+ *    segment.each_token do |token|
+ *      token => {text:, probability:}
+        puts "#{text} (#{probability})"
+ *    end
+ */
+static VALUE ruby_whisper_token_deconstruct_keys(VALUE self, VALUE keys)
+{
+  ruby_whisper_token *rwt;
+  GetToken(self, rwt);
+  VALUE hash = rb_hash_new();
+  long n_keys = 0;
+
+  if (NIL_P(keys)) {
+    keys = rb_ary_new3(
+      N_KEY_NAMES,
+      sym_id,
+      sym_tid,
+      sym_probability,
+      sym_log_probability,
+      sym_pt,
+      sym_ptsum,
+      sym_t_dtw,
+      sym_voice_length,
+      sym_start_time,
+      sym_end_time,
+      sym_text
+    );
+    n_keys = N_KEY_NAMES;
+  } else {
+    n_keys = RARRAY_LEN(keys);
+    if (n_keys > N_KEY_NAMES) {
+      return hash;
+    }
+  }
+
+  for (int i = 0; i < n_keys; i++) {
+    VALUE key = rb_ary_entry(keys, i);
+    if (key == sym_start_time) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_start_time(self));
+      continue;
+    }
+    if (key == sym_end_time) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_end_time(self));
+      continue;
+    }
+    if (key == sym_text) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_text(self));
+      continue;
+    }
+    if (key == sym_probability) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_p(self));
+      continue;
+    }
+    if (key == sym_id) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_id(self));
+      continue;
+    }
+    if (key == sym_tid) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_tid(self));
+      continue;
+    }
+    if (key == sym_log_probability) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_plog(self));
+      continue;
+    }
+    if (key == sym_pt) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_pt(self));
+      continue;
+    }
+    if (key == sym_ptsum) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_ptsum(self));
+      continue;
+    }
+    if (key == sym_t_dtw) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_t_dtw(self));
+      continue;
+    }
+    if (key == sym_voice_length) {
+      rb_hash_aset(hash, key, ruby_whisper_token_get_vlen(self));
+      continue;
+    }
+  }
+
+  return hash;
+}
+
+
+void
+init_ruby_whisper_token(VALUE *mWhisper)
+{
+  cToken = rb_define_class_under(*mWhisper, "Token", rb_cObject);
+
+  rb_define_alloc_func(cToken, ruby_whisper_token_allocate);
+
+  sym_id = ID2SYM(rb_intern("id"));
+  sym_tid = ID2SYM(rb_intern("tid"));
+  sym_probability = ID2SYM(rb_intern("probability"));
+  sym_log_probability = ID2SYM(rb_intern("log_probability"));
+  sym_pt = ID2SYM(rb_intern("pt"));
+  sym_ptsum = ID2SYM(rb_intern("ptsum"));
+  sym_t_dtw = ID2SYM(rb_intern("t_dtw"));
+  sym_voice_length = ID2SYM(rb_intern("voice_length"));
+  sym_start_time = ID2SYM(rb_intern("start_time"));
+  sym_end_time = ID2SYM(rb_intern("end_time"));
+  sym_text = ID2SYM(rb_intern("text"));
+
+  rb_define_method(cToken, "id", ruby_whisper_token_get_id, 0);
+  rb_define_method(cToken, "tid", ruby_whisper_token_get_tid, 0);
+  rb_define_method(cToken, "probability", ruby_whisper_token_get_p, 0);
+  rb_define_method(cToken, "log_probability", ruby_whisper_token_get_plog, 0);
+  rb_define_method(cToken, "pt", ruby_whisper_token_get_pt, 0);
+  rb_define_method(cToken, "ptsum", ruby_whisper_token_get_ptsum, 0);
+  rb_define_method(cToken, "t_dtw", ruby_whisper_token_get_t_dtw, 0);
+  rb_define_method(cToken, "voice_length", ruby_whisper_token_get_vlen, 0);
+  rb_define_method(cToken, "start_time", ruby_whisper_token_get_start_time, 0);
+  rb_define_method(cToken, "end_time", ruby_whisper_token_get_end_time, 0);
+  rb_define_method(cToken, "text", ruby_whisper_token_get_text, 0);
+
+  rb_define_method(cToken, "deconstruct_keys", ruby_whisper_token_deconstruct_keys, 1);
+}
+
+#undef N_KEY_NAMES