llama_cpp 0.0.4 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e659b4cc27e9ab45b524ec3d341892b72cebdf84ccad823f24ff7e472f2ffa8
-  data.tar.gz: 790f4d2f6dc9ddf211701f6014ae91ca19e0492efd7c64eb881e66729f929544
+  metadata.gz: 00d1390f6d91449d27050952aa9dd80572fb18d6809dc64098e2e4fce79bc91e
+  data.tar.gz: 49f4422e5c8903bb83fbd69a7901ff8ed0cbfae248d6652bb93b43feac331821
 SHA512:
-  metadata.gz: 54eb4dd6c4ca9affc7b6f2da1c9dc719f60c8dc3841645cf47b8f0310ff31ad16a5bc841051663f03e962bc7424f56f1d7a1c5c5eabd03e3f5e7b706467bb0ba
-  data.tar.gz: b2a16c6be3d7f117fabea5cc837b9306b0768d9ad99104a6fa2752932d1e1a034312983a87ebfe9e3ccb1bf83257d5ce40520e049a40291c64fb2fab8663882a
+  metadata.gz: 0633f8565d940b618eb1637fa095a7bc940618100698986971274ffb943fca9fab32eb04e5a0e22bd930f18f998b7f4ca59fb0e4f7ac6210efdf7d4c44e8cc9c
+  data.tar.gz: 2e79f037ec38c415cbe9a64485b986c64e3de73c26580b0e6e8be577cec38abcc4eb9506de2bd1d0b84646f4f44b7422aafe83c0ee867028c2a223f345a294ce

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 ## [Unreleased]
 
+## [[0.0.6](https://github.com/yoshoku/llama_cpp.rb/compare/v0.0.5...v0.0.6)] - 2023-04-22
+
+- Bump bundled llama.cpp from master-315a95a to master-12b5900.
+- Add model file type constants
+- Add `model_quantize` module function to LLaMACpp.
+- Add cublas config option:
+  ```
+  $ gem install llama_cpp -- --with-cublas
+  ```
+
+## [[0.0.5](https://github.com/yoshoku/llama_cpp.rb/compare/v0.0.4...v0.0.5)] - 2023-04-20
+
+- Bump bundled llama.cpp from master-c85e03d to master-315a95a.
+- Add `apply_lora_from_file` method to LLaMACpp::Context.
+- Add `mlock_supported?` module function to LLaMACpp.
+- Add `mmap_supported?` module function to LLaMACpp.
+- Fix to not destroy original prompt in `LLaMACpp.generate` module function.
+- Add check for context initialization.
+- Add blas config options:
+  ```
+  $ gem install llama_cpp -- --with-openblas
+  ```
+  macOS:
+  ```
+  $ gem install llama_cpp -- --with-openblas --with-opt-dir=/opt/homebrew/opt/openblas
+  $ gem install llama_cpp -- --with-accelerate
+  ```
+
 ## [[0.0.4](https://github.com/yoshoku/llama_cpp.rb/compare/v0.0.3...v0.0.4)] - 2023-04-15
 
 - Bump bundled llama.cpp from master-698f7b5 to master-c85e03d.

data/README.md

@@ -20,7 +20,8 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-Prepare a quantized model file by refering to [the usage section on the llama.cpp README](https://github.com/ggerganov/llama.cpp#usage).
+Prepare the quantized model by refering to [the usage section on the llama.cpp README](https://github.com/ggerganov/llama.cpp#usage) or
+download the qunatized model, for example [ggml-vicuna-7b-4bit](https://github.com/ggerganov/llama.cpp/discussions/643#discussioncomment-5541351), from Hugging Face.
 
 ```ruby
 require 'llama_cpp'
@@ -28,7 +29,7 @@ require 'llama_cpp'
 params = LLaMACpp::ContextParams.new
 params.seed = 12
 
-context = LLaMACpp::Context.new(model_path: '/path/to/ggml-model-q4_0.bin', params: params)
+context = LLaMACpp::Context.new(model_path: '/path/to/quantized-model.bin', params: params)
 
 puts LLaMACpp.generate(context, 'Please tell me the largest city in Japan.', n_threads: 4)
 # => "There are two major cities in Japan, Tokyo and Osaka, which have about 30 million populations."

data/ext/llama_cpp/extconf.rb

@@ -15,6 +15,25 @@ if RUBY_PLATFORM.match?(/darwin|linux|bsd/) && try_compile('#include <stdio.h>',
   $CXXFLAGS << ' -pthread'
 end
 
+if with_config('openblas')
+  abort 'libopenblas is not found.' unless have_library('openblas')
+  abort 'cblas.h is not found.' unless have_header('cblas.h')
+
+  $CFLAGS << ' -DGGML_USE_OPENBLAS'
+end
+
+if with_config('accelerate')
+  abort 'Accelerate framework is not found.' unless have_framework('Accelerate')
+
+  $CFLAGS << ' -DGGML_USE_ACCELERATE'
+end
+
+if with_config('cublas')
+  $CFLAGS << ' -DGGML_USE_CUBLAS -I/usr/local/cuda/include'
+  $LDFLAGS << ' -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64'
+  $objs = %w[ggml-cuda.o ggml.o llama.o llama_cpp.o]
+end
+
 UNAME_M = RbConfig::CONFIG['build_cpu'] || RbConfig::CONFIG['host_cpu'] || RbConfig::CONFIG['target_cpu']
 
 # rubocop:disable Layout/LineLength
@@ -37,3 +56,10 @@ end
 # rubocop:enable Layout/LineLength
 
 create_makefile('llama_cpp/llama_cpp')
+
+if with_config('cublas')
+  File.open('Makefile', 'a') do |f|
+    f.puts 'ggml-cuda.o: ggml-cuda.cu ggml-cuda.h'
+    f.puts "\tnvcc -arch=native -c -o $@ $<"
+  end
+end

data/ext/llama_cpp/llama_cpp.cpp

@@ -228,6 +228,7 @@ public:
     rb_define_method(rb_cLLaMAContext, "reset_timings", RUBY_METHOD_FUNC(_llama_context_reset_timings), 0);
     rb_define_method(rb_cLLaMAContext, "free", RUBY_METHOD_FUNC(_llama_context_free), 0);
     rb_define_method(rb_cLLaMAContext, "load", RUBY_METHOD_FUNC(_llama_context_load), -1);
+    rb_define_method(rb_cLLaMAContext, "apply_lora_from_file", RUBY_METHOD_FUNC(_llama_context_apply_lora_from_file), -1);
   };
 
 private:
@@ -311,6 +312,10 @@ private:
     const int n_threads = kw_values[3] == Qundef ? 1 : NUM2INT(kw_values[3]);
 
     LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
     if (llama_eval(ptr->ctx, embd.data(), n_tokens, n_past, n_threads) != 0) {
       rb_raise(rb_eRuntimeError, "Failed to evaluate");
       return Qnil;
@@ -349,6 +354,10 @@ private:
 
     std::vector<llama_token> tokens(n_max_tokens);
     LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
     const int n = llama_tokenize(ptr->ctx, text.c_str(), tokens.data(), n_max_tokens, add_bos);
     if (n < 0) {
       rb_raise(rb_eRuntimeError, "Failed to tokenize");
@@ -449,6 +458,10 @@ private:
     }
 
     LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
     llama_token token = llama_sample_top_p_top_k(ptr->ctx, last_n_tokens_data.data(), last_n_tokens_size, top_k, top_p, temp, penalty);
 
     return INT2NUM(token);
@@ -548,6 +561,43 @@ private:
     RB_GC_GUARD(filename);
     return Qnil;
   };
+
+  static VALUE _llama_context_apply_lora_from_file(int argc, VALUE* argv, VALUE self) {
+    VALUE kw_args = Qnil;
+    ID kw_table[3] = { rb_intern("lora_path"), rb_intern("base_model_path"), rb_intern("n_threads") };
+    VALUE kw_values[3] = { Qundef, Qundef, Qundef };
+    rb_scan_args(argc, argv, ":", &kw_args);
+    rb_get_kwargs(kw_args, kw_table, 1, 2, kw_values);
+
+    if (!RB_TYPE_P(kw_values[0], T_STRING)) {
+      rb_raise(rb_eArgError, "lora_path must be a string");
+      return Qnil;
+    }
+    if (kw_values[1] != Qundef && !RB_TYPE_P(kw_values[1], T_STRING)) {
+      rb_raise(rb_eArgError, "base_model_path must be a string");
+      return Qnil;
+    }
+    if (kw_values[2] != Qundef && !RB_INTEGER_TYPE_P(kw_values[2])) {
+      rb_raise(rb_eArgError, "n_threads must be an integer");
+      return Qnil;
+    }
+
+    const char* lora_path = StringValueCStr(kw_values[0]);
+    const char* base_model_path = kw_values[1] == Qundef ? NULL : StringValueCStr(kw_values[1]);
+    const int n_threads = kw_values[2] == Qundef ? 1 : NUM2INT(kw_values[2]);
+
+    LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx != NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is already loaded");
+      return Qnil;
+    }
+
+    if (llama_apply_lora_from_file(ptr->ctx, lora_path, base_model_path, n_threads) != 0) {
+      rb_raise(rb_eRuntimeError, "Failed to apply LoRA");
+      return Qnil;
+    }
+    return Qnil;
+  };
 };
 
 const rb_data_type_t RbLLaMAContext::llama_context_type = {
@@ -562,6 +612,43 @@ const rb_data_type_t RbLLaMAContext::llama_context_type = {
 
 // module functions
 
+static VALUE rb_llama_model_quantize(int argc, VALUE* argv, VALUE self) {
+  VALUE kw_args = Qnil;
+  ID kw_table[4] = { rb_intern("input_path"), rb_intern("output_path"), rb_intern("ftype"), rb_intern("n_threads") };
+  VALUE kw_values[4] = { Qundef, Qundef, Qundef, Qundef };
+  rb_scan_args(argc, argv, ":", &kw_args);
+  rb_get_kwargs(kw_args, kw_table, 3, 1, kw_values);
+
+  if (!RB_TYPE_P(kw_values[0], T_STRING)) {
+    rb_raise(rb_eArgError, "input_path must be a string");
+    return Qnil;
+  }
+  if (!RB_TYPE_P(kw_values[1], T_STRING)) {
+    rb_raise(rb_eArgError, "output_path must be a string");
+    return Qnil;
+  }
+  if (!RB_INTEGER_TYPE_P(kw_values[2])) {
+    rb_raise(rb_eArgError, "ftype must be an integer");
+    return Qnil;
+  }
+  if (kw_values[3] != Qundef && !RB_INTEGER_TYPE_P(kw_values[3])) {
+    rb_raise(rb_eArgError, "n_threads must be an integer");
+    return Qnil;
+  }
+
+  const char* input_path = StringValueCStr(kw_values[0]);
+  const char* output_path = StringValueCStr(kw_values[1]);
+  const int ftype = NUM2INT(kw_values[2]);
+  const int n_threads = kw_values[3] == Qundef ? 1 : NUM2INT(kw_values[3]);
+
+  if (llama_model_quantize(input_path, output_path, (llama_ftype)ftype, n_threads) != 0) {
+    rb_raise(rb_eRuntimeError, "Failed to quantize model");
+    return Qnil;
+  }
+
+  return Qnil;
+}
+
 static VALUE rb_llama_token_bos(VALUE self) {
   return INT2NUM(llama_token_bos());
 }
@@ -575,14 +662,33 @@ static VALUE rb_llama_print_system_info(VALUE self) {
   return rb_utf8_str_new_cstr(result);
 }
 
+static VALUE rb_llama_mmap_supported(VALUE self) {
+  return llama_mmap_supported() ? Qtrue : Qfalse;
+}
+
+static VALUE rb_llama_mlock_supported(VALUE self) {
+  return llama_mlock_supported() ? Qtrue : Qfalse;
+}
+
 extern "C" void Init_llama_cpp(void) {
   rb_mLLaMACpp = rb_define_module("LLaMACpp");
   RbLLaMAContext::define_class(rb_mLLaMACpp);
   RbLLaMAContextParams::define_class(rb_mLLaMACpp);
 
+  rb_define_module_function(rb_mLLaMACpp, "model_quantize", rb_llama_model_quantize, -1);
   rb_define_module_function(rb_mLLaMACpp, "token_bos", rb_llama_token_bos, 0);
   rb_define_module_function(rb_mLLaMACpp, "token_eos", rb_llama_token_eos, 0);
   rb_define_module_function(rb_mLLaMACpp, "print_system_info", rb_llama_print_system_info, 0);
+  rb_define_module_function(rb_mLLaMACpp, "mmap_supported?", rb_llama_mmap_supported, 0);
+  rb_define_module_function(rb_mLLaMACpp, "mlock_supported?", rb_llama_mlock_supported, 0);
+
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_ALL_F32", INT2NUM(LLAMA_FTYPE_ALL_F32));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_F16", INT2NUM(LLAMA_FTYPE_MOSTLY_F16));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_Q4_0", INT2NUM(LLAMA_FTYPE_MOSTLY_Q4_0));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_Q4_1", INT2NUM(LLAMA_FTYPE_MOSTLY_Q4_1));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16", INT2NUM(LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_Q4_2", INT2NUM(LLAMA_FTYPE_MOSTLY_Q4_2));
+  rb_define_const(rb_mLLaMACpp, "LLAMA_FTYPE_MOSTLY_Q4_3", INT2NUM(LLAMA_FTYPE_MOSTLY_Q4_3));
 
   rb_define_const(rb_mLLaMACpp, "LLAMA_FILE_VERSION", rb_str_new2(std::to_string(LLAMA_FILE_VERSION).c_str()));
   std::stringstream ss_magic;

data/ext/llama_cpp/src/ggml-cuda.h

@@ -0,0 +1,12 @@
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+void dequantize_row_q4_0_cuda(const void * vx, float * y, int k, cudaStream_t stream);
+void dequantize_row_q4_1_cuda(const void * vx, float * y, int k, cudaStream_t stream);
+void dequantize_row_q4_2_cuda(const void * vx, float * y, int k, cudaStream_t stream);
+void dequantize_row_q4_3_cuda(const void * vx, float * y, int k, cudaStream_t stream);
+
+#ifdef  __cplusplus
+}
+#endif