llama-rb 0.1.0

data/ext/llama/llama.h

@@ -0,0 +1,152 @@
+#ifndef LLAMA_H
+#define LLAMA_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#ifdef LLAMA_SHARED
+#    if defined(_WIN32) && !defined(__MINGW32__)
+#        ifdef LLAMA_BUILD
+#            define LLAMA_API __declspec(dllexport)
+#        else
+#            define LLAMA_API __declspec(dllimport)
+#        endif
+#    else
+#        define LLAMA_API __attribute__ ((visibility ("default")))
+#    endif
+#else
+#    define LLAMA_API
+#endif
+
+#define LLAMA_FILE_VERSION 1
+#define LLAMA_FILE_MAGIC 0x67676a74 // 'ggjt' in hex
+#define LLAMA_FILE_MAGIC_UNVERSIONED 0x67676d6c // pre-versioned files
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    //
+    // C interface
+    //
+    // TODO: show sample usage
+    //
+
+    struct llama_context;
+
+    typedef int llama_token;
+
+    typedef struct llama_token_data {
+        llama_token id;  // token id
+
+        float p;     // probability of the token
+        float plog;  // log probability of the token
+
+    } llama_token_data;
+
+    typedef void (*llama_progress_callback)(float progress, void *ctx);
+
+    struct llama_context_params {
+        int n_ctx;   // text context
+        int n_parts; // -1 for default
+        int seed;    // RNG seed, 0 for random
+
+        bool f16_kv;     // use fp16 for KV cache
+        bool logits_all; // the llama_eval() call computes all logits, not just the last one
+        bool vocab_only; // only load the vocabulary, no weights
+        bool use_mlock;  // force system to keep model in RAM
+        bool embedding;  // embedding mode only
+
+        // called with a progress value between 0 and 1, pass NULL to disable
+        llama_progress_callback progress_callback;
+        // context pointer passed to the progress callback
+        void * progress_callback_user_data;
+    };
+
+    LLAMA_API struct llama_context_params llama_context_default_params();
+
+    // Various functions for loading a ggml llama model.
+    // Allocate (almost) all memory needed for the model.
+    // Return NULL on failure
+    LLAMA_API struct llama_context * llama_init_from_file(
+                             const char * path_model,
+            struct llama_context_params   params);
+
+    // Frees all allocated memory
+    LLAMA_API void llama_free(struct llama_context * ctx);
+
+    // TODO: not great API - very likely to change
+    // Returns 0 on success
+    LLAMA_API int llama_model_quantize(
+            const char * fname_inp,
+            const char * fname_out,
+                   int   itype);
+
+    // Run the llama inference to obtain the logits and probabilities for the next token.
+    // tokens + n_tokens is the provided batch of new tokens to process
+    // n_past is the number of tokens to use from previous eval calls
+    // Returns 0 on success
+    LLAMA_API int llama_eval(
+            struct llama_context * ctx,
+               const llama_token * tokens,
+                             int   n_tokens,
+                             int   n_past,
+                             int   n_threads);
+
+    // Convert the provided text into tokens.
+    // The tokens pointer must be large enough to hold the resulting tokens.
+    // Returns the number of tokens on success, no more than n_max_tokens
+    // Returns a negative number on failure - the number of tokens that would have been returned
+    // TODO: not sure if correct
+    LLAMA_API int llama_tokenize(
+            struct llama_context * ctx,
+                      const char * text,
+                     llama_token * tokens,
+                             int   n_max_tokens,
+                            bool   add_bos);
+
+    LLAMA_API int llama_n_vocab(struct llama_context * ctx);
+    LLAMA_API int llama_n_ctx  (struct llama_context * ctx);
+    LLAMA_API int llama_n_embd (struct llama_context * ctx);
+
+    // Token logits obtained from the last call to llama_eval()
+    // The logits for the last token are stored in the last row
+    // Can be mutated in order to change the probabilities of the next token
+    // Rows: n_tokens
+    // Cols: n_vocab
+    LLAMA_API float * llama_get_logits(struct llama_context * ctx);
+
+    // Get the embeddings for the input
+    // shape: [n_embd] (1-dimensional)
+    LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
+
+    // Token Id -> String. Uses the vocabulary in the provided context
+    LLAMA_API const char * llama_token_to_str(struct llama_context * ctx, llama_token token);
+
+    // Special tokens
+    LLAMA_API llama_token llama_token_bos();
+    LLAMA_API llama_token llama_token_eos();
+
+    // TODO: improve the last_n_tokens interface ?
+    LLAMA_API llama_token llama_sample_top_p_top_k(
+       struct llama_context * ctx,
+          const llama_token * last_n_tokens_data,
+                        int   last_n_tokens_size,
+                        int   top_k,
+                      float   top_p,
+                      float   temp,
+                      float   repeat_penalty);
+
+    // Performance information
+    LLAMA_API void llama_print_timings(struct llama_context * ctx);
+    LLAMA_API void llama_reset_timings(struct llama_context * ctx);
+
+    // Print system information
+    LLAMA_API const char * llama_print_system_info(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/ext/llama/model.cpp

@@ -0,0 +1,192 @@
+#include <rice/rice.hpp>
+
+#include "common.h"
+#include "llama.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+class ModelCpp
+{
+	public:
+		llama_context *ctx;
+		ModelCpp()
+		{
+			ctx = NULL;
+		}
+		void model_initialize(
+			const char *model,
+			const int32_t n_ctx,
+			const int32_t n_parts,
+			const int32_t seed,
+			const bool memory_f16,
+			const bool use_mlock
+		);
+		Rice::Object model_predict(
+			const char *prompt,
+			const int32_t n_predict
+		);
+		~ModelCpp()
+		{
+
+			if (ctx != NULL) {
+				llama_free(ctx);
+			}
+		}
+};
+
+void ModelCpp::model_initialize(
+	const char *model,     // path to model file, e.g. "models/7B/ggml-model-q4_0.bin"
+	const int32_t n_ctx,   // context size
+	const int32_t n_parts, // amount of model parts (-1 = determine from model dimensions)
+	const int32_t seed,    // RNG seed
+	const bool memory_f16, // use f16 instead of f32 for memory kv
+	const bool use_mlock   // use mlock to keep model in memory
+)
+{
+	auto lparams = llama_context_default_params();
+
+	lparams.n_ctx     = n_ctx;
+	lparams.n_parts   = n_parts;
+	lparams.seed      = seed;
+	lparams.f16_kv    = memory_f16;
+	lparams.use_mlock = use_mlock;
+
+	ctx = llama_init_from_file(model, lparams);
+}
+
+Rice::Object ModelCpp::model_predict(
+	const char *prompt,     // string used as prompt
+	const int32_t n_predict // number of tokens to predict
+)
+{
+	std::string return_val = "";
+
+    gpt_params params;
+	params.prompt = prompt;
+	params.n_predict = n_predict;
+
+    // add a space in front of the first character to match OG llama tokenizer behavior
+    params.prompt.insert(0, 1, ' ');
+
+    // tokenize the prompt
+    auto embd_inp = ::llama_tokenize(ctx, params.prompt, true);
+    const int n_ctx = llama_n_ctx(ctx);
+
+    // determine newline token
+    auto llama_token_newline = ::llama_tokenize(ctx, "\n", false);
+
+	// generate output
+	{
+		std::vector<llama_token> last_n_tokens(n_ctx);
+		std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
+
+		int n_past     = 0;
+		int n_remain   = params.n_predict;
+		int n_consumed = 0;
+
+		std::vector<llama_token> embd;
+
+		while (n_remain != 0) {
+			if (embd.size() > 0) {
+				// infinite text generation via context swapping
+				// if we run out of context:
+				// - take the n_keep first tokens from the original prompt (via n_past)
+				// - take half of the last (n_ctx - n_keep) tokens and recompute the logits in a batch
+				if (n_past + (int) embd.size() > n_ctx) {
+					const int n_left = n_past - params.n_keep;
+
+					n_past = params.n_keep;
+
+					// insert n_left/2 tokens at the start of embd from last_n_tokens
+					embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size());
+				}
+
+				if (llama_eval(ctx, embd.data(), embd.size(), n_past, params.n_threads)) {
+					throw Rice::Exception(rb_eRuntimeError, "Failed to eval");
+				}
+			}
+
+
+			n_past += embd.size();
+			embd.clear();
+
+			if ((int) embd_inp.size() <= n_consumed) {
+				// out of user input, sample next token
+				const int32_t top_k          = params.top_k;
+				const float   top_p          = params.top_p;
+				const float   temp           = params.temp;
+				const float   repeat_penalty = params.repeat_penalty;
+
+				llama_token id = 0;
+
+				{
+					auto logits = llama_get_logits(ctx);
+
+					if (params.ignore_eos) {
+						logits[llama_token_eos()] = 0;
+					}
+
+					id = llama_sample_top_p_top_k(ctx,
+							last_n_tokens.data() + n_ctx - params.repeat_last_n,
+							params.repeat_last_n, top_k, top_p, temp, repeat_penalty);
+
+					last_n_tokens.erase(last_n_tokens.begin());
+					last_n_tokens.push_back(id);
+				}
+
+				// replace end of text token with newline token when in interactive mode
+				if (id == llama_token_eos() && params.interactive && !params.instruct) {
+					id = llama_token_newline.front();
+					if (params.antiprompt.size() != 0) {
+						// tokenize and inject first reverse prompt
+						const auto first_antiprompt = ::llama_tokenize(ctx, params.antiprompt.front(), false);
+						embd_inp.insert(embd_inp.end(), first_antiprompt.begin(), first_antiprompt.end());
+					}
+				}
+
+				// add it to the context
+				embd.push_back(id);
+
+				// decrement remaining sampling budget
+				--n_remain;
+			} else {
+				// some user input remains from prompt or interaction, forward it to processing
+				while ((int) embd_inp.size() > n_consumed) {
+					embd.push_back(embd_inp[n_consumed]);
+					last_n_tokens.erase(last_n_tokens.begin());
+					last_n_tokens.push_back(embd_inp[n_consumed]);
+					++n_consumed;
+					if ((int) embd.size() >= params.n_batch) {
+						break;
+					}
+				}
+			}
+
+            for (auto id : embd) {
+				return_val += llama_token_to_str(ctx, id);
+            }
+		}
+	}
+
+	Rice::String ruby_return_val(return_val);
+	return ruby_return_val;
+}
+
+extern "C"
+void Init_model()
+{
+	Rice::Module rb_mLlama = Rice::define_module("Llama");
+	Rice::Data_Type<ModelCpp> rb_cModel =Rice::define_class_under<ModelCpp>(rb_mLlama, "Model");
+
+	rb_cModel.define_constructor(Rice::Constructor<ModelCpp>());
+	rb_cModel.define_method("initialize_cpp", &ModelCpp::model_initialize);
+	rb_cModel.define_method("predict_cpp", &ModelCpp::model_predict);
+}

data/lib/llama/model.rb

@@ -0,0 +1,86 @@
+require 'tempfile'
+
+module Llama
+  class Model
+    # move methods defined in `model.cpp` from public to private
+    private :initialize_cpp, :predict_cpp
+
+    # rubocop:disable Metrics/MethodLength
+    def self.new(
+      model,               # path to model file, e.g. "models/7B/ggml-model-q4_0.bin"
+      n_ctx: 512,          # context size
+      n_parts: -1,         # amount of model parts (-1 = determine from model dimensions)
+      seed: Time.now.to_i, # RNG seed
+      memory_f16: true,    # use f16 instead of f32 for memory kv
+      use_mlock: false     # use mlock to keep model in memory
+    )
+      instance = allocate
+
+      instance.instance_eval do
+        initialize
+
+        @model = model
+        @n_ctx = n_ctx
+        @n_parts = n_parts
+        @seed = seed
+        @memory_f16 = memory_f16
+        @use_mlock = use_mlock
+
+        capture_stderr do
+          initialize_cpp(
+            model,
+            n_ctx,
+            n_parts,
+            seed,
+            memory_f16,
+            use_mlock,
+          )
+        end
+      end
+
+      instance
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def predict(
+      prompt,        # string used as prompt
+      n_predict: 128 # number of tokens to predict
+    )
+      text = ''
+
+      capture_stderr { text = predict_cpp(prompt, n_predict) }
+
+      process_text(text)
+    end
+
+    attr_reader :model, :n_ctx, :n_parts, :seed, :memory_f16, :use_mlock, :stderr
+
+    private
+
+    def capture_stderr
+      previous = $stderr.dup
+      tmp = Tempfile.open('llama-rb-stderr')
+
+      begin
+        $stderr.reopen(tmp)
+
+        yield
+
+        tmp.rewind
+        @stderr = tmp.read
+      ensure
+        tmp.close(true)
+        $stderr.reopen(previous)
+      end
+    end
+
+    def process_text(text)
+      text = text.force_encoding(Encoding.default_external)
+
+      # remove the space that was added as a tokenizer hack in model.cpp
+      text[0] = '' if text.size.positive?
+
+      text
+    end
+  end
+end

data/lib/llama/version.rb

@@ -0,0 +1,3 @@
+module Llama
+  VERSION = '0.1.0'.freeze
+end

data/lib/llama.rb

@@ -0,0 +1,6 @@
+require_relative 'llama/version'
+require_relative '../ext/llama/model'
+require_relative 'llama/model'
+
+module Llama
+end

data/llama-rb.gemspec

@@ -0,0 +1,50 @@
+require_relative 'lib/llama'
+
+Gem::Specification.new do |spec|
+  spec.name = 'llama-rb'
+  spec.version = Llama::VERSION
+  spec.licenses = ['MIT']
+  spec.authors = ['zfletch']
+  spec.email = ['zfletch2@gmail.com']
+
+  spec.summary = 'Ruby interface for Llama'
+  spec.description = 'ggerganov/llama.cpp with Ruby hooks'
+  spec.homepage = 'https://github.com/zfletch/llama-rb'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = "#{spec.homepage}/releases"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "ext/llama/common.cpp",
+    "ext/llama/common.h",
+    "ext/llama/extconf.rb",
+    "ext/llama/ggml.c",
+    "ext/llama/ggml.h",
+    "ext/llama/llama.cpp",
+    "ext/llama/llama.h",
+    "ext/llama/model.cpp",
+    "lib/llama.rb",
+    "lib/llama/model.rb",
+    "lib/llama/version.rb",
+    "llama-rb.gemspec",
+    "llama.cpp",
+    "models/.gitkeep",
+  ]
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rice', '~> 4.0.4'
+
+  spec.extensions = %w[ext/llama/extconf.rb]
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: llama-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- zfletch
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rice
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.4
+description: ggerganov/llama.cpp with Ruby hooks
+email:
+- zfletch2@gmail.com
+executables: []
+extensions:
+- ext/llama/extconf.rb
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- ext/llama/common.cpp
+- ext/llama/common.h
+- ext/llama/extconf.rb
+- ext/llama/ggml.c
+- ext/llama/ggml.h
+- ext/llama/llama.cpp
+- ext/llama/llama.h
+- ext/llama/model.cpp
+- lib/llama.rb
+- lib/llama/model.rb
+- lib/llama/version.rb
+- llama-rb.gemspec
+- models/.gitkeep
+homepage: https://github.com/zfletch/llama-rb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/zfletch/llama-rb
+  source_code_uri: https://github.com/zfletch/llama-rb
+  changelog_uri: https://github.com/zfletch/llama-rb/releases
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Ruby interface for Llama
+test_files: []