llama_cpp 0.0.6 → 0.1.0

data/ext/llama_cpp/src/llama.h

@@ -19,9 +19,11 @@
 #    define LLAMA_API
 #endif
 
-#define LLAMA_FILE_VERSION 1
-#define LLAMA_FILE_MAGIC 0x67676a74 // 'ggjt' in hex
-#define LLAMA_FILE_MAGIC_UNVERSIONED 0x67676d6c // pre-versioned files
+#define LLAMA_FILE_VERSION           1
+#define LLAMA_FILE_MAGIC             'ggjt'
+#define LLAMA_FILE_MAGIC_UNVERSIONED 'ggml'
+#define LLAMA_SESSION_MAGIC          'ggsn'
+#define LLAMA_SESSION_VERSION        1
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,18 +41,22 @@ extern "C" {
 
     typedef struct llama_token_data {
         llama_token id;  // token id
-
+        float logit; // log-odds of the token
         float p;     // probability of the token
-        float plog;  // log probability of the token
-
     } llama_token_data;
 
+    typedef struct llama_token_data_array {
+        llama_token_data * data;
+        size_t size;
+        bool sorted;
+    } llama_token_data_array;
+
     typedef void (*llama_progress_callback)(float progress, void *ctx);
 
     struct llama_context_params {
         int n_ctx;   // text context
         int n_parts; // -1 for default
-        int seed;    // RNG seed, 0 for random
+        int seed;    // RNG seed, -1 for random
 
         bool f16_kv;     // use fp16 for KV cache
         bool logits_all; // the llama_eval() call computes all logits, not just the last one
@@ -73,7 +79,10 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_Q4_1 = 3,  // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
         LLAMA_FTYPE_MOSTLY_Q4_2 = 5,  // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_3 = 6,  // except 1d tensors
+        // LLAMA_FTYPE_MOSTLY_Q4_3 (6) support has been removed
+        LLAMA_FTYPE_MOSTLY_Q8_0 = 7,  // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q5_0 = 8,  // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q5_1 = 9,  // except 1d tensors
     };
 
     LLAMA_API struct llama_context_params llama_context_default_params();
@@ -112,22 +121,28 @@ extern "C" {
                       const char * path_base_model,
                              int   n_threads);
 
-    // Returns the KV cache that will contain the context for the
-    // ongoing prediction with the model.
-    LLAMA_API const uint8_t * llama_get_kv_cache(struct llama_context * ctx);
+    // Returns the number of tokens in the KV cache
+    LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx);
 
-    // Returns the size of the KV cache
-    LLAMA_API size_t llama_get_kv_cache_size(struct llama_context * ctx);
+    // Sets the current rng seed.
+    LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, int seed);
 
-    // Returns the number of tokens in the KV cache
-    LLAMA_API int llama_get_kv_cache_token_count(struct llama_context * ctx);
+    // Returns the maximum size in bytes of the state (rng, logits, embedding
+    // and kv_cache) - will often be smaller after compacting tokens
+    LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx);
 
-    // Sets the KV cache containing the current context for the model
-    LLAMA_API void llama_set_kv_cache(
-            struct llama_context * ctx,
-                   const uint8_t * kv_cache,
-                          size_t   n_size,
-                             int   n_token_count);
+    // Copies the state to the specified destination address.
+    // Destination needs to have allocated enough memory.
+    // Returns the number of bytes copied
+    LLAMA_API size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dest);
+
+    // Set the state reading from the specified address
+    // Returns the number of bytes read
+    LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src);
+
+    // Save/load session file
+    LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out);
+    LLAMA_API bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count);
 
     // Run the llama inference to obtain the logits and probabilities for the next token.
     // tokens + n_tokens is the provided batch of new tokens to process
@@ -152,9 +167,9 @@ extern "C" {
                              int   n_max_tokens,
                             bool   add_bos);
 
-    LLAMA_API int llama_n_vocab(struct llama_context * ctx);
-    LLAMA_API int llama_n_ctx  (struct llama_context * ctx);
-    LLAMA_API int llama_n_embd (struct llama_context * ctx);
+    LLAMA_API int llama_n_vocab(const struct llama_context * ctx);
+    LLAMA_API int llama_n_ctx  (const struct llama_context * ctx);
+    LLAMA_API int llama_n_embd (const struct llama_context * ctx);
 
     // Token logits obtained from the last call to llama_eval()
     // The logits for the last token are stored in the last row
@@ -168,21 +183,57 @@ extern "C" {
     LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
 
     // Token Id -> String. Uses the vocabulary in the provided context
-    LLAMA_API const char * llama_token_to_str(struct llama_context * ctx, llama_token token);
+    LLAMA_API const char * llama_token_to_str(const struct llama_context * ctx, llama_token token);
 
     // Special tokens
     LLAMA_API llama_token llama_token_bos();
     LLAMA_API llama_token llama_token_eos();
+    LLAMA_API llama_token llama_token_nl();
+
+    // Sampling functions
+
+    /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
+    LLAMA_API void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty);
+
+    /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
+    LLAMA_API void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float alpha_frequency, float alpha_presence);
+
+    /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
+    LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates);
+
+    /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
+    LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep = 1);
+
+    /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
+    LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep = 1);
+
+    /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
+    LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep = 1);
+
+    /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
+    LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep = 1);
+    LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp);
+
+    /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
+    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
+    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
+    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
+    /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
+    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
+    LLAMA_API llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu);
+
+    /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
+    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
+    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
+    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
+    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
+    LLAMA_API llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu);
+
+    /// @details Selects the token with the highest probability.
+    LLAMA_API llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates);
 
-    // TODO: improve the last_n_tokens interface ?
-    LLAMA_API llama_token llama_sample_top_p_top_k(
-       struct llama_context * ctx,
-          const llama_token * last_n_tokens_data,
-                        int   last_n_tokens_size,
-                        int   top_k,
-                      float   top_p,
-                      float   temp,
-                      float   repeat_penalty);
+    /// @details Randomly selects a token from the candidates based on their probabilities.
+    LLAMA_API llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates);
 
     // Performance information
     LLAMA_API void llama_print_timings(struct llama_context * ctx);

data/lib/llama_cpp/client.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module LLaMACpp
+  # Client provides a high-level interface to the LLM model.
+  class Client # rubocop:disable Metrics/ClassLength
+    # Creates a new client.
+    #
+    # @param model_path [String] The path to the model file.
+    # @param lora_adapter_path [String] The path to the LoRA adapter file.
+    # @param lora_base_path [String] The path to the LoRA base model file.
+    # @param n_ctx [Integer] The context size.
+    # @param n_parts [Integer] The amount of model parts (-1 = determine from model dimensions).
+    # @param memory_f16 [Boolean] The flag wheter to use f16 instead of f32 for memory kv.
+    # @param use_mmap [Boolean] The flag whether to use mmap.
+    # @param use_mlock [Boolean] The flag hether to use mlock.
+    # @param embedding [Boolean] The flag whether to calculate embedding.
+    # @param n_threads [Integer] The number of threads to use.
+    # @param seed [Integer] The seed for the random number generator.
+    # @return [Client]
+    # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+    def initialize(model_path:, lora_adapter_path: nil, lora_base_path: nil,
+                   n_ctx: 512, n_parts: -1, memory_f16: false, use_mmap: true, use_mlock: false,
+                   embedding: false,
+                   n_threads: 1, seed: 0)
+      @params = {
+        model_path: model_path,
+        lora_adapter_path: lora_adapter_path,
+        lora_base_path: lora_base_path,
+        n_ctx: n_ctx,
+        n_parts: n_parts,
+        memory_f16: memory_f16,
+        use_mmap: use_mmap,
+        use_mlock: use_mlock,
+        embedding: embedding,
+        n_threads: n_threads,
+        seed: seed
+      }
+      @context_params = ContextParams.new
+      @context_params.n_ctx = n_ctx
+      @context_params.n_parts = n_parts
+      @context_params.f16_kv = memory_f16
+      @context_params.use_mmap = use_mmap
+      @context_params.use_mlock = use_mlock
+      @context_params.embedding = embedding
+      @context_params.seed = seed
+      @context = Context.new(model_path: model_path, params: @context_params)
+      return unless lora_adapter_path.is_a?(String)
+
+      if lora_base_path.is_a?(String)
+        @context.apply_lora_from_file(lora_path: lora_adapter_path, base_model_path: lora_base_path, n_threads: n_threads)
+      else
+        @context.apply_lora_from_file(lora_path: lora_adapter_path, n_threads: n_threads)
+      end
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
+
+    # Generates completions for a given prompt.
+    #
+    # @param prompt [String] The prompt to generate completions for.
+    # @param max_tokens [Integer] The maximum number of tokens to generate.
+    # @param n_keep [Integer] The number of tokens to keep from the initial prompt.
+    # @param repeat_last_n [Integer] The number of tokens to use for repeat penalty.
+    # @param n_batch [Integer] The batch size.
+    # @param frequency [Float] The frequency penalty value.
+    # @param presence [Float] The presence penalty value.
+    # @param top_k [Integer] The top-k value.
+    # @param top_p [Float] The top-p value.
+    # @param tfs_z [Float] The tail free sampling parameter.
+    # @param typical_p [Float] The typical probability value.
+    # @param temperature [Float] The temperature value.
+    # @param repeat_penalty [Float] The repeat penalty value.
+    # @return [String]
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
+    def completions(prompt, max_tokens: 64, n_keep: 10, repeat_last_n: 64, n_batch: 512,
+                    frequency: 0.0, presence: 0.0,
+                    top_k: 40, top_p: 0.95, tfs_z: 1.0, typical_p: 1.0, temperature: 0.8, repeat_penalty: 1.1)
+      embd_input = tokenize_prompt(prompt)
+
+      n_ctx = @context.n_ctx
+      raise ArgumentError, "prompt is too long #{embd_input.size} tokens, maximum is #{n_ctx - 4}" if embd_input.size > n_ctx - 4
+
+      last_n_tokens = [0] * n_ctx
+
+      embd = []
+      n_consumed = 0
+      n_past = 0
+      n_remain = max_tokens
+      n_vocab = @context.n_vocab
+      output = []
+
+      while n_remain != 0
+        unless embd.empty?
+          if n_past + embd.size > n_ctx
+            n_left = n_past - n_keep
+            n_past = n_keep
+            embd.insert(0, last_n_tokens[(n_ctx - (n_left / 2) - embd.size)...-embd.size])
+          end
+
+          @context.eval(tokens: embd, n_past: n_past, n_threads: @params[:n_threads])
+        end
+
+        n_past += embd.size
+        embd.clear
+
+        if embd_input.size <= n_consumed
+          logits = @context.logits
+          base_candidates = Array.new(n_vocab) { |i| LLaMACpp::TokenData.new(id: i, logit: logits[i], p: 0.0) }
+          candidates = LLaMACpp::TokenDataArray.new(base_candidates)
+
+          # apply penalties
+          last_n_repeat = [last_n_tokens.size, repeat_last_n, n_ctx].min
+          @context.sample_repetition_penalty(candidates, last_n_tokens[-last_n_repeat..], penalty: repeat_penalty)
+          @context.sample_frequency_and_presence_penalties(
+            candidates, last_n_tokens[-last_n_repeat..], frequency: frequency, presence: presence
+          )
+
+          # temperature sampling
+          @context.sample_top_k(candidates, k: top_k)
+          @context.sample_tail_free(candidates, z: tfs_z)
+          @context.sample_typical(candidates, prob: typical_p)
+          @context.sample_top_p(candidates, prob: top_p)
+          @context.sample_temperature(candidates, temperature: temperature)
+          id = @context.sample_token(candidates)
+
+          last_n_tokens.shift
+          last_n_tokens.push(id)
+
+          last_n_tokens.shift
+          last_n_tokens.push(id)
+
+          embd.push(id)
+          n_remain -= 1
+        else
+          while embd_input.size > n_consumed
+            embd.push(embd_input[n_consumed])
+            last_n_tokens.shift
+            last_n_tokens.push(embd_input[n_consumed])
+            n_consumed += 1
+            break if embd.size >= n_batch
+          end
+        end
+
+        embd.each { |token| output << @context.token_to_str(token) }
+
+        break if !embd.empty? && embd[-1] == LLaMACpp.token_eos
+      end
+
+      output.join.delete_prefix(" #{prompt}").strip
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
+
+    # def chat(prompt); end
+
+    # Obtains the embedding for a given text.
+    #
+    # @param text [String] The text to obtain the embedding for.
+    # @return [Array<Float>]
+    def embeddings(text)
+      raise 'The embedding option is set to false' unless @params[:embedding]
+
+      embd_input = tokenize_prompt(text)
+      raise 'The result of tokenizing the input text is empty' unless embd_input.size.positive?
+
+      @context.eval(tokens: embd_input, n_past: 0, n_threads: @params[:n_threads])
+      @context.embeddings
+    end
+
+    private
+
+    def tokenize_prompt(prompt)
+      @context.tokenize(text: " #{prompt}", add_bos: true)
+    end
+  end
+end

data/lib/llama_cpp/version.rb

@@ -3,8 +3,8 @@
 # llama_cpp.rb provides Ruby bindings for the llama.cpp.
 module LLaMACpp
   # The version of llama_cpp.rb you install.
-  VERSION = '0.0.6'
+  VERSION = '0.1.0'
 
   # The version of llama.cpp bundled with llama_cpp.rb.
-  LLAMA_CPP_VERSION = 'master-12b5900'
+  LLAMA_CPP_VERSION = 'master-173d0e6'
 end

data/lib/llama_cpp.rb

@@ -2,6 +2,7 @@
 
 require_relative 'llama_cpp/version'
 require_relative 'llama_cpp/llama_cpp'
+require_relative 'llama_cpp/client'
 
 # llama_cpp.rb provides Ruby bindings for the llama.cpp.
 module LLaMACpp
@@ -12,24 +13,40 @@ module LLaMACpp
 
   # Generates sentences following the given prompt for operation check.
   #
-  # @param context [LLaMACpp::Context]
-  # @param prompt [String]
-  # @param n_threads [Integer]
+  # @param context [LLaMACpp::Context] The context to use.
+  # @param prompt [String] The prompt to start generation with.
+  # @param n_predict [Integer] The number of tokens to predict.
+  # @param n_threads [Integer] The number of threads.
   # @return [String]
-  def generate(context, prompt, n_threads: 1) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/PerceivedComplexity
-    spaced_prompt = " #{prompt}"
+  def generate(context, prompt, n_predict: 128, n_threads: 1) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    raise ArgumentError, 'context must be an instance of LLaMACpp::Context' unless context.is_a?(LLaMACpp::Context)
+    raise ArgumentError, 'context must have loaded the model' if context.empty?
+    raise ArgumentError, 'prompt must be a String' unless prompt.is_a?(String)
 
+    spaced_prompt = " #{prompt}"
     embd_input = context.tokenize(text: spaced_prompt, add_bos: true)
 
     n_ctx = context.n_ctx
+    raise ArgumentError, "prompt is too long #{embd_input.size} tokens, maximum is #{n_ctx - 4}" if embd_input.size > n_ctx - 4
+
     last_n_tokens = [0] * n_ctx
 
     embd = []
     n_consumed = 0
     n_keep = 10
     n_past = 0
-    n_remain = 128
+    n_remain = n_predict
     repeat_last_n = 64
+    repeat_penalty = 1.1
+    frequency = 0.0
+    presence = 0.0
+    top_k = 40
+    top_p = 0.95
+    tfs_z = 1.0
+    typical_p = 1.0
+    temperature = 0.8
+    n_batch = 512
+    n_vocab = context.n_vocab
     output = []
 
     while n_remain != 0
@@ -47,10 +64,25 @@ module LLaMACpp
       embd.clear
 
       if embd_input.size <= n_consumed
-        start = n_ctx - repeat_last_n
-        id = context.sample_top_p_top_k(
-          last_n_tokens[start...(start + repeat_last_n)], top_k: 40, top_p: 0.95, temp: 0.80, penalty: 1.1
+        logits = context.logits
+        base_candidates = Array.new(n_vocab) { |i| LLaMACpp::TokenData.new(id: i, logit: logits[i], p: 0.0) }
+        candidates = LLaMACpp::TokenDataArray.new(base_candidates)
+
+        # apply penalties
+        last_n_repeat = [last_n_tokens.size, repeat_last_n, n_ctx].min
+        context.sample_repetition_penalty(candidates, last_n_tokens[-last_n_repeat..], penalty: repeat_penalty)
+        context.sample_frequency_and_presence_penalties(
+          candidates, last_n_tokens[-last_n_repeat..], frequency: frequency, presence: presence
         )
+
+        # temperature sampling
+        context.sample_top_k(candidates, k: top_k)
+        context.sample_tail_free(candidates, z: tfs_z)
+        context.sample_typical(candidates, prob: typical_p)
+        context.sample_top_p(candidates, prob: top_p)
+        context.sample_temperature(candidates, temperature: temperature)
+        id = context.sample_token(candidates)
+
         last_n_tokens.shift
         last_n_tokens.push(id)
 
@@ -62,13 +94,13 @@ module LLaMACpp
           last_n_tokens.shift
           last_n_tokens.push(embd_input[n_consumed])
           n_consumed += 1
-          break if embd.size >= 512
+          break if embd.size >= n_batch
         end
       end
 
       embd.each { |token| output << context.token_to_str(token) }
 
-      break if embd[-1] == LLaMACpp.token_eos
+      break if !embd.empty? && embd[-1] == LLaMACpp.token_eos
     end
 
     output.join.delete_prefix(spaced_prompt).strip

data/sig/llama_cpp.rbs

@@ -11,22 +11,46 @@ module LLaMACpp
   LLAMA_FTYPE_MOSTLY_Q4_1: Integer
   LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16: Integer
   LLAMA_FTYPE_MOSTLY_Q4_2: Integer
-  LLAMA_FTYPE_MOSTLY_Q4_3: Integer
+  LLAMA_FTYPE_MOSTLY_Q8_0: Integer
+  LLAMA_FTYPE_MOSTLY_Q5_0: Integer
+  LLAMA_FTYPE_MOSTLY_Q5_1: Integer
 
   def self?.model_quantize: (input_path: String, output_path: String, ftype: Integer, ?n_threads: Integer) -> void
-  def self?.generate: (::LLaMACpp::Context, String, ?n_threads: Integer) -> String
+  def self?.generate: (::LLaMACpp::Context, String, ?n_predict: Integer, ?n_threads: Integer) -> String
   def self?.print_system_info: () -> void
   def self?.token_bos: () -> Integer
   def self?.token_eos: () -> Integer
+  def self?.token_nl: () -> Integer
   def self?.mmap_supported?: () -> bool
   def self?.mlock_supported?: () -> bool
 
+  class TokenData
+    public
+
+    def initialize: (id: Integer, logit: Float, p: Float) -> void
+    def id: () -> Integer
+    def id=: (Integer) -> Integer
+    def logit: () -> Float
+    def logit=: (Float) -> Float
+    def p: () -> Float
+    def p=: (Float) -> Float
+  end
+
+  class TokenDataArray
+    public
+
+    def initialize: (Array[::LLaMACpp::TokenData], ?sorted: bool) -> void
+    def size: () -> Integer
+    def sorted: () -> bool
+  end
+
   class Context
     public
 
     def initialize: (model_path: String, params: ::LLaMACpp::ContextParams) -> void
                   | () -> void
     def embeddings: () -> Array[Float]
+    def empty?: () -> bool
     def eval: (tokens: Array[Integer], n_past: Integer, ?n_tokens: Integer, ?n_threads: Integer) -> void
     def free: () -> void
     def load: (model_path: String, params: ::LLaMACpp::ContextParams) -> void
@@ -36,10 +60,23 @@ module LLaMACpp
     def n_vocab: () -> Integer
     def print_timings: () -> void
     def reset_timings: () -> void
-    def sample_top_p_top_k: (top_k: Integer, top_p: Float, temp: Float, penalty: Float) -> Integer
     def token_to_str: (Integer) -> String
     def tokenize: (text: String, ?n_max_tokens: Integer, ?add_bos: bool) -> Array[Integer]
     def apply_lora_from_file: (lora_path: String, ?base_model_path: String, ?n_threads: Integer) -> void
+    def kv_cache_token_count: () -> Integer
+    def set_rng_seed: (Integer) -> void
+    def sample_repetition_penalty: (::LLaMACpp::TokenDataArray, Array[Integer], penalty: Float) -> void
+    def sample_frequency_and_presence_penalties: (::LLaMACpp::TokenDataArray, Array[Integer], frequency: Float, presence: Float) -> void
+    def sample_softmax: (::LLaMACpp::TokenDataArray) -> void
+    def sample_top_k: (::LLaMACpp::TokenDataArray, k: Integer, ?min_keep: Integer) -> void
+    def sample_top_p: (::LLaMACpp::TokenDataArray, prob: Float, ?min_keep: Integer) -> void
+    def sample_tail_free: (::LLaMACpp::TokenDataArray, z: Float, ?min_keep: Integer) -> void
+    def sample_typical: (::LLaMACpp::TokenDataArray, prob: Float, ?min_keep: Integer) -> void
+    def sample_temperature: (::LLaMACpp::TokenDataArray, temperature: Float) -> void
+    def sample_token_mirostat: (::LLaMACpp::TokenDataArray, tau: Float, eta: Float, m: Integer, mu: Float) -> [Integer, Float]
+    def sample_token_mirostat_v2: (::LLaMACpp::TokenDataArray, tau: Float, eta: Float, mu: Float) -> [Integer, Float]
+    def sample_token_greedy: (::LLaMACpp::TokenDataArray) -> Integer
+    def sample_token: (::LLaMACpp::TokenDataArray) -> Integer
   end
 
   class ContextParams
@@ -59,9 +96,22 @@ module LLaMACpp
     def seed=: (Integer) -> Integer
     def use_mlock: () -> bool
     def use_mlock=: (bool) -> bool
+    def use_mmap: () -> bool
+    def use_mmap=: (bool) -> bool
     def vocab_only: () -> bool
     def vocab_only=: (bool) -> bool
   end
 
   class Params = ContextParams
+
+  class Client
+    def initialize(model_path: String, ?lora_adapter_path: String, ?lora_base_path: String,
+                   ?n_ctx: Integer, ?n_parts: Integer, ?memory_f16: bool, ?use_mmap: bool, ?use_mlock: bool,
+                   ?embedding: bool, ?n_threads: Integer, ?seed: Integer) -> void
+    def completions(String, ?max_tokens: Integer, ?n_keep: Integer, ?repeat_last_n: Integer, ?n_batch: Integer,
+                    ?frequency: Float, ?presence: Float,
+                    ?top_k: Integer, ?top_p: Float, ?tfs_z: Float, ?typical_p: Float, ?temperature: Float,
+                    ?repeat_penalty: Float) -> String
+    def embeddings(String) -> Array[Float]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llama_cpp
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-04-22 00:00:00.000000000 Z
+date: 2023-05-20 00:00:00.000000000 Z
 dependencies: []
 description: llama_cpp.rb provides Ruby bindings for the llama.cpp.
 email:
@@ -27,12 +27,15 @@ files:
 - ext/llama_cpp/llama_cpp.h
 - ext/llama_cpp/src/LICENSE
 - ext/llama_cpp/src/ggml-cuda.h
+- ext/llama_cpp/src/ggml-opencl.c
+- ext/llama_cpp/src/ggml-opencl.h
 - ext/llama_cpp/src/ggml.c
 - ext/llama_cpp/src/ggml.h
+- ext/llama_cpp/src/llama-util.h
 - ext/llama_cpp/src/llama.cpp
 - ext/llama_cpp/src/llama.h
-- ext/llama_cpp/src/llama_util.h
 - lib/llama_cpp.rb
+- lib/llama_cpp/client.rb
 - lib/llama_cpp/version.rb
 - sig/llama_cpp.rbs
 homepage: https://github.com/yoshoku/llama_cpp.rb