llama_cpp 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e0152eb9e091932225356614b57fad416c2aa96a83316f8585c9ef2872e1504
-  data.tar.gz: 8ea2f00f11be7dd6524bfe69e3181fc63df7c841ed1e2d91b1b2bcafd99d0b66
+  metadata.gz: f1fcd28849baae5e90c466665aff4fe5da1d848193ebcf74c3fe333c5674191c
+  data.tar.gz: fcb0c64528d24c5cfad677f17bfd6e1e817a4b8279317ca5b2113302735598b9
 SHA512:
-  metadata.gz: a85a4bdd2d1fd575eb406b9bebdf7f388db33dc42f7a2980ba9a7a6b346b539854d9df5515c9b6968727e76f035a23f59d4bc65bc5525df962dfbdf56d8b3b01
-  data.tar.gz: 33641d622102257dbc1358bde0871a03c595928f5d8cedee512e1df414e4aa93433eadfcd082d4db42046320c1ed7f806dfb3aafd7934a1becb33fe275f9435c
+  metadata.gz: c70b5f919feb7a585efbe21b3360254c2f5789504cd73fecee12fd686483c77eeb763ed91a8e7434d5852208555a78f168b358d0895f15b1ea7e774d36d6910a
+  data.tar.gz: f554ad58fc9d68c39b80995b7f424468386b32a5847dbdefbceb1cba53ff7182da35be8599523d82a6daa8fee23667d07e06faedc4c727d52e8fc594d0bc7d3f

data/CHANGELOG.md

@@ -1,3 +1,37 @@
+## [[0.3.2](https://github.com/yoshoku/llama_cpp.rb/compare/v0.3.1...v0.3.2)] - 2023-07-08
+
+- Bump bundled llama.cpp from master-b8c8dda to master-481f793.
+- Add `Timings` class and `timings` method to `Context`:
+  ```ruby
+  require 'llama_cpp'
+
+  # ...
+
+  context = LLaMACpp::Context.new(model: model)
+  timings = context.timings
+
+  puts timings.class
+  # => LLaMACpp::Timings
+  puts timings.t_load_ms
+  # => 79.61
+  ```
+- Expose sampling options as the arguemnts of `generate` module function:
+  ```ruby
+  require 'llama_cpp'
+
+  # ...
+
+  LLaMACpp.generate(context, 'Hello, world.', top_k: 30, top_p: 0.8, temperature: 0.9)
+  ```
+- Add `ModelQuantizaParams` class, this class was not published because the author forgot to write rb_define_class.
+- Minor update to example scripts, configuration files, and documentations.
+
+## [[0.3.1](https://github.com/yoshoku/llama_cpp.rb/compare/v0.3.0...v0.3.1)] - 2023-07-02
+
+- Bump bundled llama.cpp from master-9d23589 to master-b8c8dda.
+  - Use unsigned values for random seed.
+- Add `eval_embd` method to `Context` class.
+
 ## [[0.3.0](https://github.com/yoshoku/llama_cpp.rb/compare/v0.2.2...v0.3.0)] - 2023-06-30
 
 - Add no_k_quants and qkk_64 config options:

data/README.md

@@ -68,6 +68,15 @@ User:
 
 ![llama_cpp_chat_example](https://github.com/yoshoku/llama_cpp.rb/assets/5562409/374ae3d8-63a6-498f-ae6e-5552b464bdda)
 
+Japanse chat is also possible using the [Vicuna model on Hugging Face](https://huggingface.co/CRD716/ggml-vicuna-1.1-quantized).
+
+```sh
+$ wget https://huggingface.co/CRD716/ggml-vicuna-1.1-quantized/resolve/main/ggml-vicuna-7b-1.1-q4_0.bin
+$ ruby chat.rb --model ggml-vicuna-7b-1.1-q4_0.bin --file prompt_jp.txt
+```
+
+![llama_cpp rb-jpchat](https://github.com/yoshoku/llama_cpp.rb/assets/5562409/526ff18c-2bb2-4b06-8933-f72960024033)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yoshoku/llama_cpp.rb.

data/examples/chat.rb

@@ -33,7 +33,7 @@ class Chat < Thor # rubocop:disable Metrics/ClassLength, Style/Documentation
   option :n_gpu_layers, type: :numeric, desc: 'number of layers on GPU', default: 0
   def main # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     params = LLaMACpp::ContextParams.new
-    params.seed = options[:seed]
+    params.seed = options[:seed] if options[:seed] != -1
     params.n_gpu_layers = options[:n_gpu_layers]
     model = LLaMACpp::Model.new(model_path: options[:model], params: params)
     context = LLaMACpp::Context.new(model: model)

data/examples/embedding.rb

@@ -18,7 +18,7 @@ class Embedding < Thor # rubocop:disable Style/Documentation
   option :n_gpu_layers, type: :numeric, desc: 'number of layers on GPU', default: 0
   def main # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     params = LLaMACpp::ContextParams.new
-    params.seed = options[:seed]
+    params.seed = options[:seed] if options[:seed] != -1
     params.n_gpu_layers = options[:n_gpu_layers]
     params.embedding = true
     model = LLaMACpp::Model.new(model_path: options[:model], params: params)

data/examples/prompt_jp.txt

@@ -0,0 +1,8 @@
+UserがTaroという名前のアシスタントと対話するダイアログのトランスクリプト。
+Taroは親切で、親切で、正直で、文章を書くのが上手で、ユーザーのリクエストに即座に正確に答えることを怠りません。
+
+User: こんにちには、Taro。
+Taro: こんにちは、今日はどのような要件ですか？
+User: 日本で最大の都市について教えてください。
+Taro: はい、日本で最大の都市は東京です。日本の首都でもあります。
+User:

data/ext/llama_cpp/extconf.rb

@@ -7,8 +7,8 @@ abort 'libstdc++ is not found.' unless have_library('stdc++')
 
 $srcs = %w[ggml.c llama.cpp llama_cpp.cpp]
 $srcs << 'ggml-opencl.cpp' if with_config('clblast')
-$CFLAGS << ' -w'
-$CXXFLAGS << ' -std=c++11'
+$CFLAGS << ' -w -DNDEBUG'
+$CXXFLAGS << ' -std=c++11 -DNDEBUG'
 $INCFLAGS << ' -I$(srcdir)/src'
 $VPATH << '$(srcdir)/src'
 

data/ext/llama_cpp/llama_cpp.cpp

@@ -1,8 +1,8 @@
-
 #include "llama_cpp.h"
 
 VALUE rb_mLLaMACpp;
 VALUE rb_cLLaMAModel;
+VALUE rb_cLLaMATimings;
 VALUE rb_cLLaMAContext;
 VALUE rb_cLLaMAContextParams;
 VALUE rb_cLLaMAModelQuantizeParams;
@@ -256,6 +256,111 @@ const rb_data_type_t RbLLaMATokenDataArray::llama_token_data_array_type = {
   RUBY_TYPED_FREE_IMMEDIATELY
 };
 
+class LLaMATimingsWrapper {
+public:
+  struct llama_timings timings;
+
+  LLaMATimingsWrapper(){};
+
+  ~LLaMATimingsWrapper(){};
+};
+
+class RbLLaMATimings {
+public:
+  static VALUE llama_timings_alloc(VALUE self) {
+    LLaMATimingsWrapper* ptr = (LLaMATimingsWrapper*)ruby_xmalloc(sizeof(LLaMATimingsWrapper));
+    new (ptr) LLaMATimingsWrapper();
+    return TypedData_Wrap_Struct(self, &llama_timings_type, ptr);
+  }
+
+  static void llama_timings_free(void* ptr) {
+    ((LLaMATimingsWrapper*)ptr)->~LLaMATimingsWrapper();
+    ruby_xfree(ptr);
+  }
+
+  static size_t llama_timings_size(const void* ptr) {
+    return sizeof(*((LLaMATimingsWrapper*)ptr));
+  }
+
+  static LLaMATimingsWrapper* get_llama_timings(VALUE self) {
+    LLaMATimingsWrapper* ptr;
+    TypedData_Get_Struct(self, LLaMATimingsWrapper, &llama_timings_type, ptr);
+    return ptr;
+  }
+
+  static void define_class(VALUE outer) {
+    rb_cLLaMATimings = rb_define_class_under(outer, "Timings", rb_cObject);
+    rb_define_alloc_func(rb_cLLaMATimings, llama_timings_alloc);
+    rb_define_method(rb_cLLaMATimings, "t_start_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_start_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "t_end_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_end_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "t_load_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_load_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "t_sample_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_sample_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "t_p_eval_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_p_eval_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "t_eval_ms", RUBY_METHOD_FUNC(_llama_timings_get_t_eval_ms), 0);
+    rb_define_method(rb_cLLaMATimings, "n_sample", RUBY_METHOD_FUNC(_llama_timings_get_n_sample), 0);
+    rb_define_method(rb_cLLaMATimings, "n_p_eval", RUBY_METHOD_FUNC(_llama_timings_get_n_p_eval), 0);
+    rb_define_method(rb_cLLaMATimings, "n_eval", RUBY_METHOD_FUNC(_llama_timings_get_n_eval), 0);
+  }
+
+private:
+  static const rb_data_type_t llama_timings_type;
+
+  static VALUE _llama_timings_get_t_start_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_start_ms);
+  }
+
+  static VALUE _llama_timings_get_t_end_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_end_ms);
+  }
+
+  static VALUE _llama_timings_get_t_load_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_load_ms);
+  }
+
+  static VALUE _llama_timings_get_t_sample_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_sample_ms);
+  }
+
+  static VALUE _llama_timings_get_t_p_eval_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_p_eval_ms);
+  }
+
+  static VALUE _llama_timings_get_t_eval_ms(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return DBL2NUM(ptr->timings.t_eval_ms);
+  }
+
+  static VALUE _llama_timings_get_n_sample(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return INT2NUM(ptr->timings.n_sample);
+  }
+
+  static VALUE _llama_timings_get_n_p_eval(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return INT2NUM(ptr->timings.n_p_eval);
+  }
+
+  static VALUE _llama_timings_get_n_eval(VALUE self) {
+    LLaMATimingsWrapper* ptr = get_llama_timings(self);
+    return INT2NUM(ptr->timings.n_eval);
+  }
+};
+
+const rb_data_type_t RbLLaMATimings::llama_timings_type = {
+  "RbLLaMATimings",
+  { NULL,
+    RbLLaMATimings::llama_timings_free,
+    RbLLaMATimings::llama_timings_size },
+  NULL,
+  NULL,
+  RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 class LLaMAContextParamsWrapper {
 public:
   struct llama_context_params params;
@@ -404,6 +509,10 @@ private:
   // seed
   static VALUE _llama_context_params_set_seed(VALUE self, VALUE seed) {
     LLaMAContextParamsWrapper* ptr = get_llama_context_params(self);
+    if (NUM2INT(seed) < 0) {
+      rb_raise(rb_eArgError, "seed must be positive");
+      return Qnil;
+    }
     ptr->params.seed = NUM2INT(seed);
     return INT2NUM(ptr->params.seed);
   };
@@ -685,6 +794,10 @@ private:
     LLaMAContextParamsWrapper* prms_ptr = RbLLaMAContextParams::get_llama_context_params(kw_values[1]);
     LLaMAModelWrapper* model_ptr = get_llama_model(self);
 
+    if (prms_ptr->params.seed == LLAMA_DEFAULT_SEED) {
+      prms_ptr->params.seed = time(NULL);
+    }
+
     try {
       model_ptr->model = llama_load_model_from_file(StringValueCStr(filename), prms_ptr->params);
     } catch (const std::runtime_error& e) {
@@ -848,6 +961,7 @@ public:
     rb_define_alloc_func(rb_cLLaMAContext, llama_context_alloc);
     rb_define_method(rb_cLLaMAContext, "initialize", RUBY_METHOD_FUNC(_llama_context_initialize), -1);
     rb_define_method(rb_cLLaMAContext, "eval", RUBY_METHOD_FUNC(_llama_context_eval), -1);
+    rb_define_method(rb_cLLaMAContext, "eval_embd", RUBY_METHOD_FUNC(_llama_context_eval_embd), -1);
     rb_define_method(rb_cLLaMAContext, "eval_export", RUBY_METHOD_FUNC(_llama_context_eval_export), 1);
     rb_define_method(rb_cLLaMAContext, "tokenize", RUBY_METHOD_FUNC(_llama_context_tokenize), -1);
     rb_define_method(rb_cLLaMAContext, "logits", RUBY_METHOD_FUNC(_llama_context_logits), 0);
@@ -857,6 +971,7 @@ public:
     rb_define_method(rb_cLLaMAContext, "n_vocab", RUBY_METHOD_FUNC(_llama_context_n_vocab), 0);
     rb_define_method(rb_cLLaMAContext, "n_ctx", RUBY_METHOD_FUNC(_llama_context_n_ctx), 0);
     rb_define_method(rb_cLLaMAContext, "n_embd", RUBY_METHOD_FUNC(_llama_context_n_embd), 0);
+    rb_define_method(rb_cLLaMAContext, "timings", RUBY_METHOD_FUNC(_llama_context_get_timings), 0);
     rb_define_method(rb_cLLaMAContext, "print_timings", RUBY_METHOD_FUNC(_llama_context_print_timings), 0);
     rb_define_method(rb_cLLaMAContext, "reset_timings", RUBY_METHOD_FUNC(_llama_context_reset_timings), 0);
     rb_define_method(rb_cLLaMAContext, "kv_cache_token_count", RUBY_METHOD_FUNC(_llama_context_kv_cache_token_count), 0);
@@ -971,6 +1086,61 @@ private:
     return Qnil;
   };
 
+  static VALUE _llama_context_eval_embd(int argc, VALUE* argv, VALUE self) {
+    VALUE kw_args = Qnil;
+    ID kw_table[4] = { rb_intern("embd"), rb_intern("n_past"), rb_intern("n_tokens"), rb_intern("n_threads") };
+    VALUE kw_values[4] = { Qundef, Qundef, Qundef, Qundef };
+    rb_scan_args(argc, argv, ":", &kw_args);
+    rb_get_kwargs(kw_args, kw_table, 2, 2, kw_values);
+
+    if (!RB_TYPE_P(kw_values[0], T_ARRAY)) {
+      rb_raise(rb_eArgError, "tokens must be an Array");
+      return Qnil;
+    }
+    if (!RB_INTEGER_TYPE_P(kw_values[1])) {
+      rb_raise(rb_eArgError, "n_past must be an integer");
+      return Qnil;
+    }
+    if (kw_values[2] != Qundef && !RB_INTEGER_TYPE_P(kw_values[2])) {
+      rb_raise(rb_eArgError, "n_tokens must be an integer");
+      return Qnil;
+    }
+    if (kw_values[3] != Qundef && !RB_INTEGER_TYPE_P(kw_values[3])) {
+      rb_raise(rb_eArgError, "n_threads must be an integer");
+      return Qnil;
+    }
+
+    const size_t tokens_len = RARRAY_LEN(kw_values[0]);
+    std::vector<float> embd(tokens_len);
+    for (size_t i = 0; i < tokens_len; i++) {
+      VALUE el = rb_ary_entry(kw_values[0], i);
+      if (!RB_FLOAT_TYPE_P(el)) {
+        rb_raise(rb_eArgError, "embd must be an array of floats");
+        return Qnil;
+      }
+      embd[i] = NUM2DBL(el);
+    }
+
+    const int n_tokens = kw_values[2] == Qundef ? (int)tokens_len : NUM2INT(kw_values[2]);
+    const int n_past = NUM2INT(kw_values[1]);
+    const int n_threads = kw_values[3] == Qundef ? 1 : NUM2INT(kw_values[3]);
+
+    LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
+    if (llama_eval_embd(ptr->ctx, embd.data(), n_tokens, n_past, n_threads) != 0) {
+      rb_raise(rb_eRuntimeError, "Failed to evaluate");
+      return Qnil;
+    }
+
+    rb_iv_set(self, "@n_tokens", INT2NUM(n_tokens));
+    rb_iv_set(self, "@has_evaluated", Qtrue);
+
+    return Qnil;
+  }
+
   static VALUE _llama_context_eval_export(VALUE self, VALUE fname_) {
     LLaMAContextWrapper* ptr = get_llama_context(self);
     if (ptr->ctx == NULL) {
@@ -1163,6 +1333,18 @@ private:
     return INT2NUM(llama_n_embd(ptr->ctx));
   };
 
+  static VALUE _llama_context_get_timings(VALUE self) {
+    LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
+    VALUE tm_obj = rb_funcall(rb_cLLaMATimings, rb_intern("new"), 0);
+    LLaMATimingsWrapper* tm_ptr = RbLLaMATimings::get_llama_timings(tm_obj);
+    tm_ptr->timings = llama_get_timings(ptr->ctx);
+    return tm_obj;
+  }
+
   static VALUE _llama_context_print_timings(VALUE self) {
     LLaMAContextWrapper* ptr = get_llama_context(self);
     if (ptr->ctx == NULL) {
@@ -1198,7 +1380,11 @@ private:
       rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
       return Qnil;
     }
-    const int seed = NUM2INT(seed_);
+    if (NUM2INT(seed_) < 0) {
+      rb_raise(rb_eArgError, "seed must be a non-negative integer");
+      return Qnil;
+    }
+    const uint32_t seed = NUM2INT(seed_);
     llama_set_rng_seed(ptr->ctx, seed);
     return Qnil;
   };
@@ -1830,8 +2016,10 @@ extern "C" void Init_llama_cpp(void) {
   RbLLaMATokenData::define_class(rb_mLLaMACpp);
   RbLLaMATokenDataArray::define_class(rb_mLLaMACpp);
   RbLLaMAModel::define_class(rb_mLLaMACpp);
+  RbLLaMATimings::define_class(rb_mLLaMACpp);
   RbLLaMAContext::define_class(rb_mLLaMACpp);
   RbLLaMAContextParams::define_class(rb_mLLaMACpp);
+  RbLLaMAModelQuantizeParams::define_class(rb_mLLaMACpp);
 
   rb_define_module_function(rb_mLLaMACpp, "init_backend", rb_llama_llama_init_backend, -1);
   rb_define_module_function(rb_mLLaMACpp, "model_quantize", rb_llama_model_quantize, -1);
@@ -1901,6 +2089,11 @@ extern "C" void Init_llama_cpp(void) {
   ss_magic << std::showbase << std::hex << LLAMA_SESSION_MAGIC;
   rb_define_const(rb_mLLaMACpp, "LLAMA_SESSION_MAGIC", rb_str_new2(ss_magic.str().c_str()));
 
+  ss_magic.str("");
+  ss_magic.clear(std::stringstream::goodbit);
+  ss_magic << std::showbase << std::hex << LLAMA_DEFAULT_SEED;
+  rb_define_const(rb_mLLaMACpp, "LLAMA_DEFAULT_SEED", rb_str_new2(ss_magic.str().c_str()));
+
   rb_define_const(rb_mLLaMACpp, "LLAMA_FILE_VERSION", rb_str_new2(std::to_string(LLAMA_FILE_VERSION).c_str()));
   rb_define_const(rb_mLLaMACpp, "LLAMA_SESSION_VERSION", rb_str_new2(std::to_string(LLAMA_SESSION_VERSION).c_str()));
 }