llama_cpp 0.1.4 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f08992d10701b3ac0ab87c32d8a28d7f81101a4896e3300c461d7015f6486814
-  data.tar.gz: 38fd72fe9cdd596f7878ef902ddf8ec8e36954bbac50388fe8ef8437a93bfe29
+  metadata.gz: ad6a2964cfc46e940026d76a3d340509ba8c30fdaf3902730081f44b7b40cfde
+  data.tar.gz: 48384234163db26b7ee45d12310ba09b1a8f4f37906ede2f9d89eb72f05df665
 SHA512:
-  metadata.gz: e3d024db508be6cbe7e644c4a9295da97742b45f921c9c5d64a7f4b4eb6be624e79f4c63d39c226566dbb9676215ae3b986828095a185cb2069547a12cf651a0
-  data.tar.gz: d884334f2d77a7204f0bc96c037fa86ee6fdf3f2879c5c5bd721be336dc743a0034733fc566c114bc6f22e620e5d79ccd4c67b6ded7d929d1949315b31445701
+  metadata.gz: 132095fecc385ca629dc051d27bafddccf893def0702690abcaf7c3b87900c643ff301bf5f3f27db99a5c58ecb90385210e35c935cf2bd99f00b2675374b31c8
+  data.tar.gz: 5987962a6d84cdf7e7a171be41e7df96a0dab94d54f408df20303d4d1622ea851c6367d9773d4d985eaa1ba77f804ab730580a1a0a4374e96b5153c1a2471ed1

data/CHANGELOG.md

@@ -1,3 +1,39 @@
+## [[0.2.1](https://github.com/yoshoku/llama_cpp.rb/compare/v0.2.0...v0.2.1)] - 2023-06-17
+
+- Bump bundled llama.cpp from master-4de0334 to master-a09f919.
+- Add `low_vram` parameter to ContextParams.
+- Add `vocab` method to Context.
+- Add example script: https://github.com/yoshoku/llama_cpp.rb/tree/main/examples
+
+## [[0.2.0](https://github.com/yoshoku/llama_cpp.rb/compare/v0.1.4...v0.2.0)] - 2023-06-11
+
+- Bump bundled llama.cpp from master-ffb06a3 to master-4de0334.
+- Fix installation files for CUDA.
+- Add metal config option:
+  ```
+  $ gem install llama_cpp -- --with-metal
+  ```
+  ```ruby
+  require 'llama_cpp'
+
+  params = LLaMACpp::ContextParams.new
+  params.n_gpu_layers = 1
+
+  context = LLaMACpp::Context.new(model_path: '/path/to/quantized-model.bin', params: params)
+  LLaMACpp.generate(context, 'Hello, world.')
+  ```
+
+**Breaking Changes**
+
+- Add ModelQuantizationParams class.
+- Change the argument of the `model_quantize` module function in LLaMACpp.
+  ```ruby
+  require 'llama_cpp'
+
+  params = LLaMACpp::ModelQuantizeParams.new
+  LLaMACpp.model_quantize(input_path: 'foo.model', output_path: 'bar.model', params: params)
+  ```
+
 ## [[0.1.4](https://github.com/yoshoku/llama_cpp.rb/compare/v0.1.3...v0.1.4)] - 2023-06-03
 
 - Bump bundled llama.cpp from master-66874d4 to master-ffb06a3.

data/examples/README.md

@@ -0,0 +1,60 @@
+# llama_cpp.rb/examples
+
+## chat.rb
+
+### Usage
+
+```sh
+$ cd examples
+$ gem install llama_cpp thor
+$ ./chat.rb -m /path/to/quantized-model.bin -t 4
+...
+User: Please tell me the largest city in Japan.
+Bob: Sure. The largest city in Japan is Tokyo.
+User:
+```
+
+### Options
+
+```sh
+$ ./chat.rb help main
+Usage:
+  chat.rb main -m, --model=MODEL
+
+Options:
+  -s, [--seed=N]                         # random seed
+                                         # Default: -1
+  -t, [--threads=N]                      # number of threads
+                                         # Default: 2
+  -m, --model=MODEL                      # path to model file
+  -f, [--file=FILE]                      # prompt file to start generation
+  -r, [--reverse-prompt=REVERSE_PROMPT]  # halt generation at PROMPT, return control in interactive mode
+  -b, [--batch-size=N]                   # batch size for prompt processing
+                                         # Default: 1024
+  -n, [--n-predict=N]                    # number of tokens to predict
+                                         # Default: 256
+      [--keep=N]                         # number of tokens to keep from the initial prompt
+                                         # Default: 48
+      [--repeat-last-n=N]                # last n tokens to consider for penalize
+                                         # Default: 64
+      [--repeat-penalty=N]               # penalize repeat sequence of tokens
+                                         # Default: 1.0
+      [--presence-penalty=N]             # repeat alpha presence penalty
+                                         # Default: 0.0
+      [--frequency-penalty=N]            # repeat alpha frequency penalty
+                                         # Default: 0.0
+      [--top-k=N]                        # top k sampling
+                                         # Default: 40
+      [--top-p=N]                        # top p sampling
+                                         # Default: 0.95
+      [--tfs-z=N]                        # tail free sampling, parameter z
+                                         # Default: 1.0
+      [--typical-p=N]                    # locally typical sampling, parameter p
+                                         # Default: 1.0
+      [--temp=N]                         # temperature
+                                         # Default: 0.8
+      [--n-gpu-layers=N]                 # number of layers on GPU
+                                         # Default: 0
+
+Start chat
+```

data/examples/chat.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# chat.rb is a simple chatbot that uses llama_cpp to generate text.
+# It is created with reference to main.cpp and chat.sh in llama.cpp examples:
+# - https://github.com/ggerganov/llama.cpp/blob/master/examples/main/main.cpp
+# - https://github.com/ggerganov/llama.cpp/blob/master/examples/chat.sh
+
+require 'llama_cpp'
+require 'thor'
+require 'readline'
+
+class Chat < Thor # rubocop:disable Metrics/ClassLength, Style/Documentation
+  default_command :main
+  desc 'main', 'Start chat'
+  option :seed, type: :numeric, aliases: '-s', desc: 'random seed', default: -1
+  option :threads, type: :numeric, aliases: '-t', desc: 'number of threads', default: 2
+  option :model, type: :string, aliases: '-m', desc: 'path to model file', required: true
+  option :file, type: :string, aliases: '-f', desc: 'prompt file to start generation'
+  option :reverse_prompt, type: :string, aliases: '-r', desc: 'halt generation at PROMPT, return control in interactive mode'
+  option :batch_size, type: :numeric, aliases: '-b', desc: 'batch size for prompt processing', default: 1024
+  option :n_predict, type: :numeric, aliases: '-n', desc: 'number of tokens to predict', default: 256
+  option :keep, type: :numeric, desc: 'number of tokens to keep from the initial prompt', default: 48
+  option :repeat_last_n, type: :numeric, desc: 'last n tokens to consider for penalize', default: 64
+  option :repeat_penalty, type: :numeric, desc: 'penalize repeat sequence of tokens', default: 1.0
+  option :presence_penalty, type: :numeric, desc: 'repeat alpha presence penalty', default: 0.0
+  option :frequency_penalty, type: :numeric, desc: 'repeat alpha frequency penalty', default: 0.0
+  option :top_k, type: :numeric, desc: 'top k sampling', default: 40
+  option :top_p, type: :numeric, desc: 'top p sampling', default: 0.95
+  option :tfs_z, type: :numeric, desc: 'tail free sampling, parameter z', default: 1.0
+  option :typical_p, type: :numeric, desc: 'locally typical sampling, parameter p', default: 1.0
+  option :temp, type: :numeric, desc: 'temperature', default: 0.8
+  option :n_gpu_layers, type: :numeric, desc: 'number of layers on GPU', default: 0
+  def main # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    params = LLaMACpp::ContextParams.new
+    params.seed = options[:seed]
+    params.n_gpu_layers = options[:n_gpu_layers]
+    context = LLaMACpp::Context.new(model_path: options[:model], params: params)
+
+    antiprompt = options[:reverse_prompt] || 'User:'
+    start_prompt = read_prompt(options[:file]) || default_prompt(antiprompt)
+
+    embd_input = context.tokenize(text: start_prompt, add_bos: true)
+
+    n_ctx = context.n_ctx
+    raise ArgumentError, "prompt is too long #{embd_input.size} tokens, maximum is #{n_ctx - 4}" if embd_input.size > n_ctx - 4
+
+    n_keep = options[:keep]
+    n_keep = embd_input.size if n_keep > embd_input.size
+
+    token_newline = context.tokenize(text: "\n", add_bos: false)
+
+    last_n_tokens = [0] * n_ctx
+    interactive = true
+    is_interacting = false
+    input_echo = true
+    first_input = true
+    embd = []
+    n_consumed = 0
+    n_past = 0
+    n_remain = options[:n_predict]
+    n_vocab = context.n_vocab
+
+    while interactive
+      unless embd.empty?
+        if n_past + embd.size > n_ctx
+          n_left = n_past - n_keep
+          n_past = [1, n_keep].max
+          embd.insert(0, last_n_tokens[(n_ctx - (n_left / 2) - embd.size)...-embd.size])
+        end
+
+        0.step(embd.size - 1, options[:batch_size]) do |i|
+          n_eval = [options[:batch_size], embd.size - i].min
+          context.eval(tokens: embd[i...i + n_eval], n_past: n_past, n_threads: options[:threads])
+          n_past += n_eval
+        end
+      end
+
+      embd.clear
+
+      if embd_input.size <= n_consumed && !is_interacting
+        logits = context.logits
+        base_candidates = Array.new(n_vocab) { |i| LLaMACpp::TokenData.new(id: i, logit: logits[i], p: 0.0) }
+        candidates = LLaMACpp::TokenDataArray.new(base_candidates)
+
+        last_n_repeat = [last_n_tokens.size, options[:repeat_last_n], n_ctx].min
+        context.sample_repetition_penalty(candidates, last_n_tokens[-last_n_repeat..], penalty: options[:repeat_penalty])
+        context.sample_frequency_and_presence_penalties(
+          candidates, last_n_tokens[-last_n_repeat..],
+          frequency: options[:frequency_penalty], presence: options[:presence_penalty]
+        )
+
+        context.sample_top_k(candidates, k: options[:top_k])
+        context.sample_tail_free(candidates, z: options[:tfs_z])
+        context.sample_typical(candidates, prob: options[:typical_p])
+        context.sample_top_p(candidates, prob: options[:top_p])
+        context.sample_temperature(candidates, temperature: options[:temp])
+        id = context.sample_token(candidates)
+
+        last_n_tokens.shift
+        last_n_tokens.push(id)
+
+        if id == LLaMACpp.token_eos
+          id = token_newline.first
+          unless antiprompt.empty?
+            first_antiprompt = context.tokenize(text: antiprompt, add_bos: false)
+            embd_input.concat(first_antiprompt)
+          end
+        end
+
+        embd.push(id)
+        input_echo = true
+        n_remain -= 1
+      else
+        while embd_input.size > n_consumed
+          embd.push(embd_input[n_consumed])
+          last_n_tokens.shift
+          last_n_tokens.push(embd_input[n_consumed])
+          n_consumed += 1
+          break if embd.size >= options[:batch_size]
+        end
+      end
+
+      if input_echo
+        output = []
+        embd.each { |token| output << context.token_to_str(token) }
+        output_str = output.join
+        output_str.chomp!(antiprompt) if first_input
+        print(output_str)
+      end
+
+      if embd_input.size <= n_consumed
+        if antiprompt.size.positive?
+          last_output = []
+          last_n_tokens.each { |token| last_output << context.token_to_str(token) }
+          last_output_str = last_output.join
+
+          search_start_pos = last_output_str.size > antiprompt.size ? last_output_str.size - antiprompt.size : 0
+          unless last_output_str.index(antiprompt, search_start_pos).nil?
+            is_interacting = true
+            true
+          end
+        end
+
+        if n_past.positive? && is_interacting
+          if first_input
+            print("\r#{antiprompt}")
+            first_input = false
+          end
+          buffer = Readline.readline(' ')
+          break interactive = false if buffer.nil?
+
+          if buffer.size > 1
+            line_input = context.tokenize(text: "#{buffer}\n", add_bos: false)
+            embd_input.concat(line_input)
+            n_remain -= line_input.size
+          end
+
+          input_echo = false
+        end
+
+        is_interacting = false if n_past.positive?
+      end
+
+      if n_remain <= 0 && options[:n_predict] != -1
+        n_remain = options[:n_predict]
+        is_interacting = true
+      end
+    end
+  end
+
+  private
+
+  def read_prompt(filename)
+    return if filename.nil?
+
+    File.read(filename).chomp
+  end
+
+  def default_prompt(antiprompt)
+    # Reference:
+    # https://github.com/ggerganov/llama.cpp/blob/master/prompts/chat-with-bob.txt
+    prompt = <<~MSG
+      Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
+
+      User: Hello, Bob.
+      Bob: Hello. How may I help you today?
+      User: Please tell me the largest city in Europe.
+      Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
+    MSG
+    prompt + antiprompt
+  end
+end
+
+Chat.start(ARGV)

data/ext/llama_cpp/extconf.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'mkmf'
+require 'fileutils'
 
 abort 'libstdc++ is not found.' unless have_library('stdc++')
 
@@ -36,17 +37,30 @@ if with_config('accelerate')
   $CFLAGS << ' -DGGML_USE_ACCELERATE'
 end
 
+if with_config('metal')
+  $CFLAGS << ' -DGGML_USE_METAL -DGGML_METAL_NDEBUG'
+  $CXXFLAGS << ' -DGGML_USE_METAL'
+  $LDFLAGS << ' -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders'
+  $objs = %w[ggml.o llama.o llama_cpp.o ggml-metal.o]
+end
+
 if with_config('cublas')
   $CFLAGS << ' -DGGML_USE_CUBLAS -I/usr/local/cuda/include'
+  $CXXFLAGS << ' -DGGML_USE_CUBLAS -I/usr/local/cuda/include'
   $LDFLAGS << ' -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64'
   $objs = %w[ggml-cuda.o ggml.o llama.o llama_cpp.o]
 end
 
 if with_config('clblast')
   abort 'libclblast is not found.' unless have_library('clblast')
-  abort 'libOpenCL is not found.' unless have_library('OpenCL')
 
   $CFLAGS << ' -DGGML_USE_CLBLAST'
+  $CXXFLAGS << ' -DGGML_USE_CLBLAST'
+  if RUBY_PLATFORM.match?(/darwin/)
+    $LDFLAGS << ' -framework OpenCL'
+  else
+    abort 'libOpenCL is not found.' unless have_library('OpenCL')
+  end
 end
 
 UNAME_M = RbConfig::CONFIG['build_cpu'] || RbConfig::CONFIG['host_cpu'] || RbConfig::CONFIG['target_cpu']
@@ -78,3 +92,14 @@ if with_config('cublas')
     f.puts "\tnvcc -arch=native -c -o $@ $<"
   end
 end
+
+if with_config('metal')
+  File.open('Makefile', 'a') do |f|
+    f.puts 'ggml-metal.o: ggml-metal.m ggml-metal.h'
+    f.puts "\t$(CC) $(CFLAGS) -c $< -o $@"
+  end
+
+  metal_path = File.expand_path("#{__dir__}/src/ggml-metal.metal")
+  dest_path = File.expand_path("#{__dir__}/../../lib/llama_cpp/")
+  FileUtils.cp(metal_path, dest_path)
+end