llama_cpp 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf8532fb7d2d96acd42b0da600cd5a8923411545817cb433036c182cb6d549ca
-  data.tar.gz: 2aa68d4ffe814632d6b8f2d97f6407284520d2b540b8920c40d486c599221bc3
+  metadata.gz: ad6a2964cfc46e940026d76a3d340509ba8c30fdaf3902730081f44b7b40cfde
+  data.tar.gz: 48384234163db26b7ee45d12310ba09b1a8f4f37906ede2f9d89eb72f05df665
 SHA512:
-  metadata.gz: ea8dad06ae15f9ca6ba585ae901d163bc6580543131338bbe785444083791b6251b2f631725190f9935740d0169520e3da604a66330e5bf7551031b7dc47dd81
-  data.tar.gz: 3b32180e6a4653af2afac59d4640c5d06b29f5872d7ef40f33dcddc27b97e2eb0fa2f38fe6389ddcb60b4631db9e3ebfd0d4b14cc9de6419e50452b4c67ad98a
+  metadata.gz: 132095fecc385ca629dc051d27bafddccf893def0702690abcaf7c3b87900c643ff301bf5f3f27db99a5c58ecb90385210e35c935cf2bd99f00b2675374b31c8
+  data.tar.gz: 5987962a6d84cdf7e7a171be41e7df96a0dab94d54f408df20303d4d1622ea851c6367d9773d4d985eaa1ba77f804ab730580a1a0a4374e96b5153c1a2471ed1

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [[0.2.1](https://github.com/yoshoku/llama_cpp.rb/compare/v0.2.0...v0.2.1)] - 2023-06-17
+
+- Bump bundled llama.cpp from master-4de0334 to master-a09f919.
+- Add `low_vram` parameter to ContextParams.
+- Add `vocab` method to Context.
+- Add example script: https://github.com/yoshoku/llama_cpp.rb/tree/main/examples
+
 ## [[0.2.0](https://github.com/yoshoku/llama_cpp.rb/compare/v0.1.4...v0.2.0)] - 2023-06-11
 
 - Bump bundled llama.cpp from master-ffb06a3 to master-4de0334.

data/examples/README.md

@@ -0,0 +1,60 @@
+# llama_cpp.rb/examples
+
+## chat.rb
+
+### Usage
+
+```sh
+$ cd examples
+$ gem install llama_cpp thor
+$ ./chat.rb -m /path/to/quantized-model.bin -t 4
+...
+User: Please tell me the largest city in Japan.
+Bob: Sure. The largest city in Japan is Tokyo.
+User:
+```
+
+### Options
+
+```sh
+$ ./chat.rb help main
+Usage:
+  chat.rb main -m, --model=MODEL
+
+Options:
+  -s, [--seed=N]                         # random seed
+                                         # Default: -1
+  -t, [--threads=N]                      # number of threads
+                                         # Default: 2
+  -m, --model=MODEL                      # path to model file
+  -f, [--file=FILE]                      # prompt file to start generation
+  -r, [--reverse-prompt=REVERSE_PROMPT]  # halt generation at PROMPT, return control in interactive mode
+  -b, [--batch-size=N]                   # batch size for prompt processing
+                                         # Default: 1024
+  -n, [--n-predict=N]                    # number of tokens to predict
+                                         # Default: 256
+      [--keep=N]                         # number of tokens to keep from the initial prompt
+                                         # Default: 48
+      [--repeat-last-n=N]                # last n tokens to consider for penalize
+                                         # Default: 64
+      [--repeat-penalty=N]               # penalize repeat sequence of tokens
+                                         # Default: 1.0
+      [--presence-penalty=N]             # repeat alpha presence penalty
+                                         # Default: 0.0
+      [--frequency-penalty=N]            # repeat alpha frequency penalty
+                                         # Default: 0.0
+      [--top-k=N]                        # top k sampling
+                                         # Default: 40
+      [--top-p=N]                        # top p sampling
+                                         # Default: 0.95
+      [--tfs-z=N]                        # tail free sampling, parameter z
+                                         # Default: 1.0
+      [--typical-p=N]                    # locally typical sampling, parameter p
+                                         # Default: 1.0
+      [--temp=N]                         # temperature
+                                         # Default: 0.8
+      [--n-gpu-layers=N]                 # number of layers on GPU
+                                         # Default: 0
+
+Start chat
+```

data/examples/chat.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# chat.rb is a simple chatbot that uses llama_cpp to generate text.
+# It is created with reference to main.cpp and chat.sh in llama.cpp examples:
+# - https://github.com/ggerganov/llama.cpp/blob/master/examples/main/main.cpp
+# - https://github.com/ggerganov/llama.cpp/blob/master/examples/chat.sh
+
+require 'llama_cpp'
+require 'thor'
+require 'readline'
+
+class Chat < Thor # rubocop:disable Metrics/ClassLength, Style/Documentation
+  default_command :main
+  desc 'main', 'Start chat'
+  option :seed, type: :numeric, aliases: '-s', desc: 'random seed', default: -1
+  option :threads, type: :numeric, aliases: '-t', desc: 'number of threads', default: 2
+  option :model, type: :string, aliases: '-m', desc: 'path to model file', required: true
+  option :file, type: :string, aliases: '-f', desc: 'prompt file to start generation'
+  option :reverse_prompt, type: :string, aliases: '-r', desc: 'halt generation at PROMPT, return control in interactive mode'
+  option :batch_size, type: :numeric, aliases: '-b', desc: 'batch size for prompt processing', default: 1024
+  option :n_predict, type: :numeric, aliases: '-n', desc: 'number of tokens to predict', default: 256
+  option :keep, type: :numeric, desc: 'number of tokens to keep from the initial prompt', default: 48
+  option :repeat_last_n, type: :numeric, desc: 'last n tokens to consider for penalize', default: 64
+  option :repeat_penalty, type: :numeric, desc: 'penalize repeat sequence of tokens', default: 1.0
+  option :presence_penalty, type: :numeric, desc: 'repeat alpha presence penalty', default: 0.0
+  option :frequency_penalty, type: :numeric, desc: 'repeat alpha frequency penalty', default: 0.0
+  option :top_k, type: :numeric, desc: 'top k sampling', default: 40
+  option :top_p, type: :numeric, desc: 'top p sampling', default: 0.95
+  option :tfs_z, type: :numeric, desc: 'tail free sampling, parameter z', default: 1.0
+  option :typical_p, type: :numeric, desc: 'locally typical sampling, parameter p', default: 1.0
+  option :temp, type: :numeric, desc: 'temperature', default: 0.8
+  option :n_gpu_layers, type: :numeric, desc: 'number of layers on GPU', default: 0
+  def main # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    params = LLaMACpp::ContextParams.new
+    params.seed = options[:seed]
+    params.n_gpu_layers = options[:n_gpu_layers]
+    context = LLaMACpp::Context.new(model_path: options[:model], params: params)
+
+    antiprompt = options[:reverse_prompt] || 'User:'
+    start_prompt = read_prompt(options[:file]) || default_prompt(antiprompt)
+
+    embd_input = context.tokenize(text: start_prompt, add_bos: true)
+
+    n_ctx = context.n_ctx
+    raise ArgumentError, "prompt is too long #{embd_input.size} tokens, maximum is #{n_ctx - 4}" if embd_input.size > n_ctx - 4
+
+    n_keep = options[:keep]
+    n_keep = embd_input.size if n_keep > embd_input.size
+
+    token_newline = context.tokenize(text: "\n", add_bos: false)
+
+    last_n_tokens = [0] * n_ctx
+    interactive = true
+    is_interacting = false
+    input_echo = true
+    first_input = true
+    embd = []
+    n_consumed = 0
+    n_past = 0
+    n_remain = options[:n_predict]
+    n_vocab = context.n_vocab
+
+    while interactive
+      unless embd.empty?
+        if n_past + embd.size > n_ctx
+          n_left = n_past - n_keep
+          n_past = [1, n_keep].max
+          embd.insert(0, last_n_tokens[(n_ctx - (n_left / 2) - embd.size)...-embd.size])
+        end
+
+        0.step(embd.size - 1, options[:batch_size]) do |i|
+          n_eval = [options[:batch_size], embd.size - i].min
+          context.eval(tokens: embd[i...i + n_eval], n_past: n_past, n_threads: options[:threads])
+          n_past += n_eval
+        end
+      end
+
+      embd.clear
+
+      if embd_input.size <= n_consumed && !is_interacting
+        logits = context.logits
+        base_candidates = Array.new(n_vocab) { |i| LLaMACpp::TokenData.new(id: i, logit: logits[i], p: 0.0) }
+        candidates = LLaMACpp::TokenDataArray.new(base_candidates)
+
+        last_n_repeat = [last_n_tokens.size, options[:repeat_last_n], n_ctx].min
+        context.sample_repetition_penalty(candidates, last_n_tokens[-last_n_repeat..], penalty: options[:repeat_penalty])
+        context.sample_frequency_and_presence_penalties(
+          candidates, last_n_tokens[-last_n_repeat..],
+          frequency: options[:frequency_penalty], presence: options[:presence_penalty]
+        )
+
+        context.sample_top_k(candidates, k: options[:top_k])
+        context.sample_tail_free(candidates, z: options[:tfs_z])
+        context.sample_typical(candidates, prob: options[:typical_p])
+        context.sample_top_p(candidates, prob: options[:top_p])
+        context.sample_temperature(candidates, temperature: options[:temp])
+        id = context.sample_token(candidates)
+
+        last_n_tokens.shift
+        last_n_tokens.push(id)
+
+        if id == LLaMACpp.token_eos
+          id = token_newline.first
+          unless antiprompt.empty?
+            first_antiprompt = context.tokenize(text: antiprompt, add_bos: false)
+            embd_input.concat(first_antiprompt)
+          end
+        end
+
+        embd.push(id)
+        input_echo = true
+        n_remain -= 1
+      else
+        while embd_input.size > n_consumed
+          embd.push(embd_input[n_consumed])
+          last_n_tokens.shift
+          last_n_tokens.push(embd_input[n_consumed])
+          n_consumed += 1
+          break if embd.size >= options[:batch_size]
+        end
+      end
+
+      if input_echo
+        output = []
+        embd.each { |token| output << context.token_to_str(token) }
+        output_str = output.join
+        output_str.chomp!(antiprompt) if first_input
+        print(output_str)
+      end
+
+      if embd_input.size <= n_consumed
+        if antiprompt.size.positive?
+          last_output = []
+          last_n_tokens.each { |token| last_output << context.token_to_str(token) }
+          last_output_str = last_output.join
+
+          search_start_pos = last_output_str.size > antiprompt.size ? last_output_str.size - antiprompt.size : 0
+          unless last_output_str.index(antiprompt, search_start_pos).nil?
+            is_interacting = true
+            true
+          end
+        end
+
+        if n_past.positive? && is_interacting
+          if first_input
+            print("\r#{antiprompt}")
+            first_input = false
+          end
+          buffer = Readline.readline(' ')
+          break interactive = false if buffer.nil?
+
+          if buffer.size > 1
+            line_input = context.tokenize(text: "#{buffer}\n", add_bos: false)
+            embd_input.concat(line_input)
+            n_remain -= line_input.size
+          end
+
+          input_echo = false
+        end
+
+        is_interacting = false if n_past.positive?
+      end
+
+      if n_remain <= 0 && options[:n_predict] != -1
+        n_remain = options[:n_predict]
+        is_interacting = true
+      end
+    end
+  end
+
+  private
+
+  def read_prompt(filename)
+    return if filename.nil?
+
+    File.read(filename).chomp
+  end
+
+  def default_prompt(antiprompt)
+    # Reference:
+    # https://github.com/ggerganov/llama.cpp/blob/master/prompts/chat-with-bob.txt
+    prompt = <<~MSG
+      Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
+
+      User: Hello, Bob.
+      Bob: Hello. How may I help you today?
+      User: Please tell me the largest city in Europe.
+      Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
+    MSG
+    prompt + antiprompt
+  end
+end
+
+Chat.start(ARGV)

data/ext/llama_cpp/llama_cpp.cpp

@@ -300,6 +300,8 @@ public:
     rb_define_method(rb_cLLaMAContextParams, "main_gpu=", RUBY_METHOD_FUNC(_llama_context_params_set_main_gpu), 1);
     rb_define_method(rb_cLLaMAContextParams, "main_gpu", RUBY_METHOD_FUNC(_llama_context_params_get_main_gpu), 0);
     rb_define_method(rb_cLLaMAContextParams, "tensor_split", RUBY_METHOD_FUNC(_llama_context_params_get_tensor_split), 0);
+    rb_define_method(rb_cLLaMAContextParams, "low_vram=", RUBY_METHOD_FUNC(_llama_context_params_set_low_vram), 1);
+    rb_define_method(rb_cLLaMAContextParams, "low_vram", RUBY_METHOD_FUNC(_llama_context_params_get_low_vram), 0);
     rb_define_method(rb_cLLaMAContextParams, "seed=", RUBY_METHOD_FUNC(_llama_context_params_set_seed), 1);
     rb_define_method(rb_cLLaMAContextParams, "seed", RUBY_METHOD_FUNC(_llama_context_params_get_seed), 0);
     rb_define_method(rb_cLLaMAContextParams, "f16_kv=", RUBY_METHOD_FUNC(_llama_context_params_set_f16_kv), 1);
@@ -386,6 +388,18 @@ private:
     return ret;
   };
 
+  // low_vram
+  static VALUE _llama_context_params_set_low_vram(VALUE self, VALUE low_vram) {
+    LLaMAContextParamsWrapper* ptr = get_llama_context_params(self);
+    ptr->params.low_vram = low_vram == Qtrue ? true : false;
+    return ptr->params.low_vram ? Qtrue : Qfalse;
+  };
+
+  static VALUE _llama_context_params_get_low_vram(VALUE self) {
+    LLaMAContextParamsWrapper* ptr = get_llama_context_params(self);
+    return ptr->params.low_vram ? Qtrue : Qfalse;
+  };
+
   // seed
   static VALUE _llama_context_params_set_seed(VALUE self, VALUE seed) {
     LLaMAContextParamsWrapper* ptr = get_llama_context_params(self);
@@ -641,6 +655,7 @@ public:
     rb_define_method(rb_cLLaMAContext, "tokenize", RUBY_METHOD_FUNC(_llama_context_tokenize), -1);
     rb_define_method(rb_cLLaMAContext, "logits", RUBY_METHOD_FUNC(_llama_context_logits), 0);
     rb_define_method(rb_cLLaMAContext, "embeddings", RUBY_METHOD_FUNC(_llama_context_embeddings), 0);
+    rb_define_method(rb_cLLaMAContext, "vocab", RUBY_METHOD_FUNC(_llama_context_vocab), -1);
     rb_define_method(rb_cLLaMAContext, "token_to_str", RUBY_METHOD_FUNC(_llama_context_token_to_str), 1);
     rb_define_method(rb_cLLaMAContext, "n_vocab", RUBY_METHOD_FUNC(_llama_context_n_vocab), 0);
     rb_define_method(rb_cLLaMAContext, "n_ctx", RUBY_METHOD_FUNC(_llama_context_n_ctx), 0);
@@ -896,6 +911,43 @@ private:
     return output;
   };
 
+  static VALUE _llama_context_vocab(int argc, VALUE* argv, VALUE self) {
+    VALUE kw_args = Qnil;
+    ID kw_table[1] = { rb_intern("capacity") };
+    VALUE kw_values[1] = { Qundef };
+    rb_scan_args(argc, argv, ":", &kw_args);
+    rb_get_kwargs(kw_args, kw_table, 1, 0, kw_values);
+
+    if (!RB_INTEGER_TYPE_P(kw_values[0])) {
+      rb_raise(rb_eArgError, "capacity must be an integer");
+      return Qnil;
+    }
+
+    LLaMAContextWrapper* ptr = get_llama_context(self);
+    if (ptr->ctx == NULL) {
+      rb_raise(rb_eRuntimeError, "LLaMA context is not initialized");
+      return Qnil;
+    }
+
+    const int capacity = NUM2INT(kw_values[0]);
+    std::vector<const char*> strings;
+    std::vector<float> scores;
+    int n_vocab = llama_n_vocab(ptr->ctx);
+    strings.resize(n_vocab, NULL);
+    scores.resize(n_vocab, 0);
+
+    n_vocab = llama_get_vocab(ptr->ctx, strings.data(), scores.data(), capacity);
+
+    VALUE ret_strings = rb_ary_new();
+    VALUE ret_scores = rb_ary_new();
+    for (int i = 0; i < n_vocab; i++) {
+      rb_ary_push(ret_strings, rb_utf8_str_new_cstr(strings[i]));
+      rb_ary_push(ret_scores, DBL2NUM(static_cast<double>(scores[i])));
+    }
+
+    return rb_ary_new_from_args(2, ret_strings, ret_scores);
+  };
+
   static VALUE _llama_context_n_vocab(VALUE self) {
     LLaMAContextWrapper* ptr = get_llama_context(self);
     if (ptr->ctx == NULL) {