llama_cpp 0.5.3 → 0.6.0

data/lib/llama_cpp.rb

@@ -5,9 +5,6 @@ require_relative 'llama_cpp/llama_cpp'
 
 # llama_cpp.rb provides Ruby bindings for the llama.cpp.
 module LLaMACpp
-  # Class alias to match interface of whispercpp gem.
-  Params = ContextParams
-
   module_function
 
   # Generates sentences following the given prompt for operation check.
@@ -15,7 +12,6 @@ module LLaMACpp
   # @param context [LLaMACpp::Context] The context to use.
   # @param prompt [String] The prompt to start generation with.
   # @param n_predict [Integer] The number of tokens to predict.
-  # @param n_threads [Integer] The number of threads.
   # @param n_keep [Integer] The number of tokens to keep in the context.
   # @param n_batch [Integer] The number of tokens to process in a batch.
   # @param repeat_last_n [Integer] The number of tokens to consider for repetition penalty.
@@ -29,14 +25,14 @@ module LLaMACpp
   # @param temperature [Float] The temperature for temperature sampling.
   # @return [String]
   def generate(context, prompt, # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
-               n_predict: 128, n_threads: 1, n_keep: 10, n_batch: 512, repeat_last_n: 64,
+               n_predict: 128, n_keep: 10, n_batch: 512, repeat_last_n: 64,
                repeat_penalty: 1.1, frequency: 0.0, presence: 0.0, top_k: 40,
                top_p: 0.95, tfs_z: 1.0, typical_p: 1.0, temperature: 0.8)
     raise ArgumentError, 'context must be an instance of LLaMACpp::Context' unless context.is_a?(LLaMACpp::Context)
     raise ArgumentError, 'prompt must be a String' unless prompt.is_a?(String)
 
     spaced_prompt = " #{prompt}"
-    embd_input = context.tokenize(text: spaced_prompt, add_bos: true)
+    embd_input = context.model.tokenize(text: spaced_prompt, add_bos: true)
 
     n_ctx = context.n_ctx
     raise ArgumentError, "prompt is too long #{embd_input.size} tokens, maximum is #{n_ctx - 4}" if embd_input.size > n_ctx - 4
@@ -47,7 +43,7 @@ module LLaMACpp
     n_consumed = 0
     n_past = 0
     n_remain = n_predict
-    n_vocab = context.n_vocab
+    n_vocab = context.model.n_vocab
     output = []
 
     while n_remain != 0
@@ -58,7 +54,7 @@ module LLaMACpp
           embd.insert(0, last_n_tokens[(n_ctx - (n_left / 2) - embd.size)...-embd.size])
         end
 
-        context.eval(tokens: embd, n_past: n_past, n_threads: n_threads)
+        context.eval(tokens: embd, n_past: n_past)
       end
 
       n_past += embd.size
@@ -99,7 +95,7 @@ module LLaMACpp
         end
       end
 
-      embd.each { |token| output << context.token_to_piece(token) }
+      embd.each { |token| output << context.model.token_to_piece(token) }
 
       break if !embd.empty? && embd[-1] == context.token_eos
     end

data/sig/llama_cpp.rbs

@@ -67,14 +67,13 @@ module LLaMACpp
   class Model
     public
 
-    def initialize: (model_path: String, params: ::LLaMACpp::ContextParams) -> void
+    def initialize: (model_path: String, params: ::LLaMACpp::ModelParams) -> void
                   | () -> void
     def empty?: () -> bool
     def free: () -> void
-    def load: (model_path: String, params: ::LLaMACpp::ContextParams) -> void
-    def apply_lora_from_file: (lora_path: String, ?base_model_path: String, ?n_threads: Integer) -> void
+    def load: (model_path: String, params: ::LLaMACpp::ModelParams) -> void
+    def apply_lora_from_file: (lora_path: String, ?scale: Float, ?base_model_path: String, ?n_threads: Integer) -> void
     def n_vocab: () -> Integer
-    def n_ctx: () -> Integer
     def n_ctx_train: () -> Integer
     def n_embd: () -> Integer
     def token_to_piece: (Integer) -> String
@@ -98,10 +97,50 @@ module LLaMACpp
     def n_eval: () -> Integer
   end
 
+  class ModelParams
+    public
+
+    def n_gpu_layers: () -> Integer
+    def n_gpu_layers=: (Integer) -> Integer
+    def main_gpu: () -> Integer
+    def main_gpu=: (Integer) -> Integer
+    def tensor_split: () -> Array[Float]
+    def vocab_only: () -> bool
+    def vocab_only=: (bool) -> bool
+    def use_mmap: () -> bool
+    def use_mmap=: (bool) -> bool
+    def use_mlock: () -> bool
+    def use_mlock=: (bool) -> bool
+  end
+
+  class Batch
+    public
+
+    def initialize: (n_tokens: Integer, embd: Integer) -> void
+    def n_tokens=: (Integer) -> Integer
+    def n_tokens: () -> Integer
+    def all_pos_zero=: (Integer) -> Integer
+    def all_pos_zero: () -> Integer
+    def all_pos_one=: (Integer) -> Integer
+    def all_pos_one: () -> Integer
+    def all_seq_id=: (Integer) -> Integer
+    def all_seq_id: () -> Integer
+    def set_token: (Integer, Integer) -> Integer
+    def get_token: (Integer) -> Integer
+    def set_pos: (Integer, Integer) -> Integer
+    def get_pos: (Integer) -> Integer
+    def set_seq_id: (Integer, Integer) -> Integer
+    def get_seq_id: (Integer) -> Integer
+    def set_logit: (Integer, bool) -> bool
+    def get_logit: (Integer) -> bool
+  end
+
   class Context
     public
 
-    def initialize: (model: ::LLaMACpp::Model) -> void
+    attr_reader model: ::LLaMACpp::Model
+
+    def initialize: (model: ::LLaMACpp::Model, params: ::LLaMACpp::ContextParams) -> void
     def embeddings: () -> Array[Float]
     def text: (Integer) -> String
     def score: (Integer) -> Float
@@ -109,20 +148,20 @@ module LLaMACpp
     def token_bos: () -> Integer
     def token_eos: () -> Integer
     def token_nl: () -> Integer
-    def eval: (tokens: Array[Integer], n_past: Integer, ?n_tokens: Integer, ?n_threads: Integer) -> void
-    def eval_embd: (tokens: Array[Float], n_past: Integer, ?n_tokens: Integer, ?n_threads: Integer) -> void
-    def eval_export: (String) -> bool
+    def eval: (tokens: Array[Integer], n_past: Integer, ?n_tokens: Integer) -> void
+    def eval_embd: (tokens: Array[Float], n_past: Integer, ?n_tokens: Integer) -> void
+    def decode: (::LLaMACpp::Batch) -> void
     def logits: () -> Array[Float]
     def n_ctx: () -> Integer
-    def n_ctx_train: () -> Integer
-    def n_embd: () -> Integer
-    def n_vocab: () -> Integer
     def timings: () -> ::LLaMACpp::Timings
     def print_timings: () -> void
     def reset_timings: () -> void
-    def token_to_piece: (Integer) -> String
-    def tokenize: (text: String, ?n_max_tokens: Integer, ?add_bos: bool) -> Array[Integer]
     def kv_cache_token_count: () -> Integer
+    def kv_cache_tokens_rm: (Integer, Integer) -> void
+    def kv_cache_seq_rm: (Integer, Integer,Integer) -> void
+    def kv_cache_seq_cp: (Integer, Integer,Integer, Integer) -> void
+    def kv_cache_seq_keep: (Integer) -> void
+    def kv_cache_seq_shift: (Integer, Integer, Ingteger, Integer) -> void
     def set_rng_seed: (Integer) -> void
     def load_session_file: (session_path: String) -> void
     def save_session_file: (session_path: String, session_tokens: Array[Integer]) -> void
@@ -134,6 +173,7 @@ module LLaMACpp
     def sample_top_p: (::LLaMACpp::TokenDataArray, prob: Float, ?min_keep: Integer) -> void
     def sample_tail_free: (::LLaMACpp::TokenDataArray, z: Float, ?min_keep: Integer) -> void
     def sample_typical: (::LLaMACpp::TokenDataArray, prob: Float, ?min_keep: Integer) -> void
+    def sample_temp: (::LLaMACpp::TokenDataArray, temp: Float) -> void
     def sample_temperature: (::LLaMACpp::TokenDataArray, temperature: Float) -> void
     def sample_token_mirostat: (::LLaMACpp::TokenDataArray, tau: Float, eta: Float, m: Integer, mu: Float) -> [Integer, Float]
     def sample_token_mirostat_v2: (::LLaMACpp::TokenDataArray, tau: Float, eta: Float, mu: Float) -> [Integer, Float]
@@ -146,37 +186,28 @@ module LLaMACpp
   class ContextParams
     public
 
-    def embedding: () -> bool
-    def embedding=: (bool) -> bool
-    def f16_kv: () -> bool
-    def f16_kv=: (bool) -> bool
-    def logits_all: () -> bool
-    def logits_all=: (bool) -> bool
+    def seed: () -> Integer
+    def seed=: (Integer) -> Integer
     def n_ctx: () -> Integer
     def n_ctx=: (Integer) -> Integer
     def n_batch: () -> Integer
     def n_batch=: (Integer) -> Integer
-    def n_gpu_layers: () -> Integer
-    def n_gpu_layers=: (Integer) -> Integer
-    def main_gpu: () -> Integer
-    def main_gpu=: (Integer) -> Integer
-    def tensor_split: () -> Array[Float]
+    def n_threads: () -> Integer
+    def n_threads=: (Integer) -> Integer
+    def n_threads_batch: () -> Integer
+    def n_threads_batch=: (Integer) -> Integer
     def rope_freq_base=: (Float) -> Float
     def rope_freq_base: () -> Float
     def rope_freq_scale=: (Float) -> Float
     def rope_freq_scale: () -> Float
-    def low_vram: () -> bool
-    def low_vram=: (bool) -> bool
     def mul_mat_q: () -> bool
     def mul_mat_q=: (bool) -> bool
-    def seed: () -> Integer
-    def seed=: (Integer) -> Integer
-    def use_mlock: () -> bool
-    def use_mlock=: (bool) -> bool
-    def use_mmap: () -> bool
-    def use_mmap=: (bool) -> bool
-    def vocab_only: () -> bool
-    def vocab_only=: (bool) -> bool
+    def f16_kv: () -> bool
+    def f16_kv=: (bool) -> bool
+    def logits_all: () -> bool
+    def logits_all=: (bool) -> bool
+    def embedding: () -> bool
+    def embedding=: (bool) -> bool
   end
 
   class ModelQuantizeParams

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llama_cpp
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-22 00:00:00.000000000 Z
+date: 2023-09-30 00:00:00.000000000 Z
 dependencies: []
 description: llama_cpp.rb provides Ruby bindings for the llama.cpp.
 email:
@@ -75,7 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Ruby bindings for the llama.cpp.