rllama 1.0.0-x86_64-linux-gnu

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 117ba4498d285d83158050dec220dab571fde8abe9f62ca9a1da07c07c1750f2
+  data.tar.gz: 0e2a7bc873398ad490d3f90b9488b808bd197544c98747f916be9d7d9914ce1f
+SHA512:
+  metadata.gz: 128cde36b03ced401b2d7f3a9ceb84c00ef91381368e4f9b8eba2714e99af4d552e102d8a23f300440e25b20712f2b81233f8c254ec44cb82232a88fd84093ca
+  data.tar.gz: 77609e417bae1d4c2b8d9e2351ce753b1adefe03f81ca761a3ad59013fecb856aaa7d1ab1c6e2a2f9a9031a057cd474f1f1c63b31d3d5dfa7802efdd636f73d7

data/README.md

@@ -0,0 +1,214 @@
+# Rllama
+
+Ruby bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) to run open-source language models locally. Run models like GPT-OSS, Qwen 3, Gemma 3, Llama 3, and many others directly in your Ruby application code.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rllama'
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+Or install it yourself as:
+
+```bash
+gem install rllama
+```
+
+## Usage
+
+### Text Generation
+
+Generate text completions using local language models:
+
+```ruby
+require 'rllama'
+
+# Load a model
+model = Rllama.load_model('lmstudio-community/gemma-3-1B-it-QAT-GGUF/gemma-3-1B-it-QAT-Q4_0.gguf')
+
+# Generate text
+result = model.generate('What is the capital of France?')
+puts result.text
+# => "The capital of France is Paris."
+
+# Access generation statistics
+puts "Tokens generated: #{result.stats[:tokens_generated]}"
+puts "Tokens per second: #{result.stats[:tps]}"
+puts "Duration: #{result.stats[:duration]} seconds"
+
+# Don't forget to close the model when done
+model.close
+```
+
+#### Generation parameters
+
+Adjust the generation with parameters:
+
+```ruby
+result = model.generate(
+  'Write a short poem about Ruby programming',
+  max_tokens: 2024,
+  temperature: 0.8,
+  top_k: 40,
+  top_p: 0.95,
+  min_p: 0.05
+)
+```
+
+#### Streaming generation
+
+Stream generated text token-by-token:
+
+```ruby
+model.generate('Explain quantum computing') do |token|
+  print token
+end
+```
+
+#### System prompt
+
+Include system promt to guide model behavior:
+
+```ruby
+result = model.generate(
+  'What are best practices for Ruby development?',
+  system: 'You are an expert Ruby developer with 10 years of experience.'
+)
+```
+
+#### Messages list
+
+Pass multiple messages with roles for more complex interactions:
+
+```ruby
+result = model.generate([
+  { role: 'system', content: 'You are a helpful assistant.' },
+  { role: 'user', content: 'What is the capital of France?' },
+  { role: 'assistant', content: 'The capital of France is Paris.' },
+  { role: 'user', content: 'What is its population?' }
+])
+puts result.text
+```
+
+### Chat
+
+For ongoing conversations, use a context object that maintains the conversation history:
+
+```ruby
+# Initialize a chat context
+context = model.init_context
+
+# Send messages and maintain conversation history
+response1 = context.message('What is the capital of France?')
+puts response1.text
+# => "The capital of France is Paris."
+
+response2 = context.message('What is the population of that city?')
+puts response2.text
+# => "Paris has a population of approximately 2.1 million people..."
+
+response3 = context.message('What was my first message?')
+puts response3.text
+# => "Your first message was asking about the capital of France."
+
+# The context remembers all previous messages in the conversation
+
+# Close context when done
+context.close
+```
+
+### Embeddings
+
+Generate vector embeddings for text using embedding models:
+
+```ruby
+require 'rllama'
+
+# Load an embedding model
+model = Rllama.load_model('lmstudio-community/embeddinggemma-300m-qat-GGUF/embeddinggemma-300m-qat-Q4_0.gguf')
+
+# Generate embedding for a single text
+embedding = model.embed('Hello, world!')
+puts embedding.length
+# => 724 (depending on your model)
+
+# Generate embeddings for multiple sentences
+embeddings = model.embed([
+  'roses are red',
+  'violets are blue',
+  'sugar is sweet'
+])
+
+puts embeddings.length
+# => 3
+puts embeddings[0].length
+# => 768
+
+model.close
+```
+
+#### Vector parameters
+
+By default, embedding vectors are normalized. You can disable normalization with `normalize: false`:
+
+```ruby
+# Generate unnormalized embeddings
+embedding = model.embed('Sample text', normalize: false)
+
+# Use custom batch size for processing multiple texts
+embeddings = model.embed(
+  ['roses are red', 'violets are blue', 'sugar is sweet'],
+  normalize: true
+)
+```
+
+## CLI Chat Utility
+
+The `rllama` command-line utility provides an interactive chat interface for conversing with language models. After installing the gem, you can start chatting immediately:
+
+```bash
+rllama
+```
+
+When you run `rllama` without arguments, it will display:
+
+- **Downloaded models**: Any models you've already downloaded to `~/.rllama/models/`
+- **Popular models**: A curated list of popular models available for download, including:
+  - Gemma 3 1B
+  - Llama 3.2 3B
+  - Phi-4
+  - Qwen3 30B
+  - GPT-OSS
+  - And more...
+
+Simply enter the number of the model you want to use. If you select a model that hasn't been downloaded yet, it will be automatically downloaded from Hugging Face.
+
+You can also specify a model path directly:
+
+```bash
+rllama path/to/your/model.gguf
+```
+
+Once the model loads, you can start chatting.
+
+## Finding Models
+
+You can download GGUF format models from various sources:
+
+- [Hugging Face](https://huggingface.co/models?library=gguf) - Search for models with "GGUF" format
+
+## License
+
+MIT
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/docusealco/rllama.

data/bin/rllama

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'rllama'
+require 'rllama/cli'
+
+Rllama::Cli.start(ARGV)

data/lib/rllama/cli.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'readline'
+
+module Rllama
+  class Cli
+    POPULAR_MODELS = [
+      { path: 'lmstudio-community/gemma-3-1B-it-QAT-GGUF/gemma-3-1B-it-QAT-Q4_0.gguf', size: 720_425_472 },
+      { path: 'lmstudio-community/gpt-oss-20b-GGUF/gpt-oss-20b-MXFP4.gguf', size: 12_109_565_632 },
+      { path: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q4_K_M.gguf', size: 2_019_377_696 },
+      { path: 'unsloth/Qwen3-30B-A3B-GGUF/Qwen3-30B-A3B-Q3_K_S.gguf', size: 13_292_468_800 },
+      { path: 'inclusionAI/Ling-mini-2.0-GGUF/Ling-mini-2.0-Q4_K_M.gguf', size: 9_911_575_072 },
+      { path: 'unsloth/gemma-3n-E4B-it-GGUF/gemma-3n-E4B-it-Q4_K_S.gguf', size: 4_404_697_216 },
+      { path: 'microsoft/phi-4-gguf/phi-4-Q4_K_S.gguf', size: 8_440_762_560 }
+    ].freeze
+
+    COLOR_CODES = {
+      red: 31,
+      green: 32,
+      yellow: 33,
+      blue: 34,
+      magenta: 35,
+      cyan: 36
+    }.freeze
+
+    def self.start(args)
+      new(args).run
+    end
+
+    def initialize(args)
+      @args = args
+
+      @model_path = args.first
+    end
+
+    def run
+      model_path = select_or_load_model
+
+      puts "\n#{colorize('Loading model...', :yellow)}"
+
+      model = Rllama.load_model(model_path)
+      context = model.init_context
+
+      puts colorize('Model loaded successfully!', :green)
+      puts "\n#{colorize('Chat started. Type your message and press Enter. Type "exit" or "quit" to end the chat.',
+                         :cyan)}\n\n"
+
+      chat_loop(context)
+    rescue Interrupt
+      puts "\n\n#{colorize('Chat interrupted. Goodbye!', :yellow)}"
+      exit(0)
+    rescue StandardError => e
+      puts "\n#{colorize("Error: #{e.message}", :red)}"
+      exit(1)
+    ensure
+      context&.close
+      model&.close
+    end
+
+    private
+
+    def select_or_load_model
+      return @model_path if @model_path
+
+      downloaded_models = find_downloaded_models
+      downloaded_model_names = downloaded_models.map { |path| File.basename(path, '.gguf') }
+
+      available_popular = POPULAR_MODELS.reject do |popular_model|
+        popular_filename = File.basename(popular_model[:path], '.gguf')
+        downloaded_model_names.any?(popular_filename)
+      end
+
+      all_choices = []
+      current_index = 1
+
+      unless downloaded_models.empty?
+        puts "#{colorize('Downloaded models:', :cyan)}\n\n"
+
+        downloaded_models.each do |model|
+          display_name = File.basename(model, '.gguf')
+          size = format_file_size(File.size(model))
+          puts "  #{colorize(current_index.to_s, :green)}. #{display_name} #{colorize("(#{size})", :yellow)}"
+          all_choices << model
+          current_index += 1
+        end
+
+        puts "\n"
+      end
+
+      unless available_popular.empty?
+        puts "#{colorize('Popular models (not downloaded):', :cyan)}\n\n"
+
+        available_popular.each do |model|
+          display_name = File.basename(model[:path], '.gguf')
+          puts "  #{colorize(current_index.to_s, :green)}. " \
+               "#{display_name} #{colorize("(#{format_file_size(model[:size])})", :yellow)}"
+          all_choices << model[:path]
+          current_index += 1
+        end
+
+        puts "\n"
+      end
+
+      if all_choices.empty?
+        puts colorize('No models available', :yellow)
+        exit(1)
+      end
+
+      print colorize("Enter number (1-#{all_choices.length}): ", :cyan)
+
+      choice = $stdin.gets&.strip.to_i
+
+      if choice < 1 || choice > all_choices.length
+        puts colorize('Invalid choice', :red)
+
+        exit(1)
+      end
+
+      all_choices[choice - 1]
+    end
+
+    def find_downloaded_models
+      models_dir = File.join(Dir.home, '.rllama', 'models')
+
+      return [] unless Dir.exist?(models_dir)
+
+      Dir.glob(File.join(models_dir, '**', '*.gguf')).reject do |path|
+        basename = File.basename(path)
+
+        basename.start_with?('~', '!')
+      end
+    end
+
+    def format_file_size(bytes)
+      gb = bytes / (1024.0**3)
+
+      if gb >= 1.0
+        format('%.1fGB', gb)
+      else
+        mb = bytes / (1024.0**2)
+
+        format('%dMB', mb.round)
+      end
+    end
+
+    def chat_loop(context)
+      loop do
+        user_input = Readline.readline('> ', false)&.strip
+
+        break if user_input.nil?
+
+        next if user_input.empty?
+
+        if user_input.downcase == 'exit' || user_input.downcase == 'quit'
+          puts "\n#{colorize('Goodbye!', :yellow)}"
+
+          break
+        end
+
+        puts "\n"
+
+        print "#{colorize('Assistant:', :magenta, bold: true)} "
+
+        context.generate(user_input) do |token|
+          print token
+          $stdout.flush
+        end
+
+        puts "\n\n"
+      end
+    end
+
+    def colorize(text, color, bold: false)
+      return text unless $stdout.tty?
+
+      code = COLOR_CODES[color] || 37
+
+      prefix = bold ? "\e[1;#{code}m" : "\e[#{code}m"
+
+      "#{prefix}#{text}\e[0m"
+    end
+  end
+end

data/lib/rllama/context.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+module Rllama
+  class Context
+    attr_reader :messages, :n_ctx, :n_batch, :n_past
+
+    def initialize(model, embeddings: false, n_ctx: nil, n_batch: nil)
+      @model = model
+      @n_ctx = n_ctx
+      @n_batch = n_batch
+      @embeddings = embeddings
+
+      @ctx_params = Cpp.llama_context_default_params
+
+      @ctx_params[:n_ctx] = @n_ctx
+      @ctx_params[:n_batch] = @n_batch
+
+      if @embeddings
+        @ctx_params[:n_seq_max] = [@n_batch, @model.n_seq_max].min
+        @ctx_params[:embeddings] = true
+      end
+
+      @pointer = Cpp.llama_init_from_model(model.pointer, @ctx_params)
+
+      raise Error, 'Failed to create the llama_context' if @pointer.null?
+
+      @n_past = 0
+      @messages = []
+    end
+
+    def generate(message, role: 'user', max_tokens: @n_ctx, temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05,
+                 seed: nil, system: nil)
+      @messages << { role: 'system', content: system } if system && @messages.empty?
+
+      if message.is_a?(Array)
+        @messages.push(*message)
+      elsif message.is_a?(Hash)
+        @messages.push(message)
+      else
+        @messages << { role: role, content: message }
+      end
+
+      prompt_string = @model.build_chat_template(@messages)
+
+      n_prompt_tokens = -Cpp.llama_tokenize(@model.vocab, prompt_string, prompt_string.bytesize, nil, 0, true, true)
+
+      raise Error, 'Prompt is too long.' if n_prompt_tokens.negative?
+
+      prompt_tokens_ptr = FFI::MemoryPointer.new(:int32, n_prompt_tokens)
+      tokens_written = Cpp.llama_tokenize(@model.vocab, prompt_string, prompt_string.bytesize, prompt_tokens_ptr,
+                                          n_prompt_tokens, true, true)
+
+      raise Error, 'Failed to tokenize prompt.' if tokens_written.negative?
+
+      new_token_count = tokens_written - @n_past
+
+      if new_token_count.positive?
+        new_tokens_ptr = prompt_tokens_ptr + (@n_past * FFI.type_size(:int32))
+
+        batch = Cpp.llama_batch_get_one(new_tokens_ptr, new_token_count)
+
+        raise Error, 'llama_decode failed.' if Cpp.llama_decode(@pointer, batch) != 0
+
+        @n_past = tokens_written
+      end
+
+      chain_params = Cpp.llama_sampler_chain_default_params
+      sampler_chain = Cpp.llama_sampler_chain_init(chain_params)
+
+      Cpp.llama_sampler_chain_add(sampler_chain, Cpp.llama_sampler_init_min_p(min_p, 1)) if min_p
+      Cpp.llama_sampler_chain_add(sampler_chain, Cpp.llama_sampler_init_top_k(top_k)) if top_k&.positive?
+      Cpp.llama_sampler_chain_add(sampler_chain, Cpp.llama_sampler_init_top_p(top_p, 1)) if top_p && top_p < 1.0
+      if temperature&.positive?
+        Cpp.llama_sampler_chain_add(sampler_chain,
+                                    Cpp.llama_sampler_init_temp(temperature))
+      end
+
+      is_probabilistic = temperature&.positive? || top_k&.positive? || (top_p && top_p < 1.0) || !min_p.nil?
+      rng_seed = seed || (Random.new_seed & 0xFFFFFFFF)
+
+      if is_probabilistic
+        Cpp.llama_sampler_chain_add(sampler_chain, Cpp.llama_sampler_init_dist(rng_seed))
+      else
+        Cpp.llama_sampler_chain_add(sampler_chain, Cpp.llama_sampler_init_greedy)
+      end
+
+      n_decoded = 0
+
+      generated_text = ''.b
+
+      assistant_message = { role: 'assistant', content: generated_text }
+
+      @messages << assistant_message
+
+      start_time = Time.now
+
+      loop do
+        break if n_decoded >= max_tokens
+
+        new_token_id = Cpp.llama_sampler_sample(sampler_chain, @pointer, -1)
+
+        break if Cpp.llama_vocab_is_eog(@model.vocab, new_token_id)
+
+        buffer = FFI::MemoryPointer.new(:char, 256)
+        n_chars = Cpp.llama_token_to_piece(@model.vocab, new_token_id, buffer, buffer.size, 0, true)
+
+        if n_chars >= 0
+          piece_bytes = buffer.read_string(n_chars)
+          utf8_piece = piece_bytes.force_encoding(Encoding::UTF_8)
+          generated_text << utf8_piece
+          yield utf8_piece if block_given?
+        end
+
+        token_ptr = FFI::MemoryPointer.new(:int32, 1).put_int32(0, new_token_id)
+        batch = Cpp.llama_batch_get_one(token_ptr, 1)
+
+        raise Error, 'context length has been exceeded' if @n_past >= @n_ctx
+        raise Error, 'llama_decode failed.' if Cpp.llama_decode(@pointer, batch) != 0
+
+        @n_past += 1
+        n_decoded += 1
+      end
+
+      end_time = Time.now
+
+      duration = end_time - start_time
+
+      tps = n_decoded.positive? && duration.positive? ? n_decoded / duration : 0
+
+      Cpp.llama_sampler_free(sampler_chain)
+
+      Result.new(
+        text: generated_text,
+        stats: {
+          duration:,
+          tokens_generated: n_decoded,
+          tps:,
+          seed: rng_seed
+        }
+      )
+    end
+    alias message generate
+
+    def embed(strings, normalize: true, batch_size: 512)
+      is_array = strings.is_a?(Array)
+
+      strings = Array(strings) unless is_array
+
+      tokenized_strings = strings.map do |text|
+        max_tokens = text.bytesize + 2
+        tokens_ptr = FFI::MemoryPointer.new(:int32, max_tokens)
+        count = Cpp.llama_tokenize(@model.vocab, text, text.bytesize, tokens_ptr, max_tokens, true, false)
+
+        raise Error, "Failed to tokenize text: '#{text}'" if count.negative?
+
+        tokens_ptr.read_array_of_int32(count)
+      end
+
+      all_embeddings = []
+      batch = Cpp.llama_batch_init(batch_size, 0, 1)
+      prompts_in_batch = []
+      current_batch_token_count = 0
+
+      process_batch = lambda do
+        next if prompts_in_batch.empty?
+
+        batch[:n_tokens] = current_batch_token_count
+
+        raise Error, 'llama_decode failed' unless Cpp.llama_decode(@pointer, batch).zero?
+
+        prompts_in_batch.each do |seq_id_in_batch|
+          embd_ptr = Cpp.llama_get_embeddings_seq(@pointer, seq_id_in_batch)
+
+          raise Error, 'Failed to get embedding' if embd_ptr.null?
+
+          embedding = embd_ptr.read_array_of_float(@model.n_embd)
+
+          all_embeddings << (normalize ? normalize_embedding(embedding) : embedding)
+        end
+
+        current_batch_token_count = 0
+        prompts_in_batch = []
+      end
+
+      tokenized_strings.each do |tokens|
+        batch_full = (current_batch_token_count + tokens.size) > batch_size
+        seq_limit_reached = prompts_in_batch.size >= @model.n_seq_max
+        process_batch.call if !prompts_in_batch.empty? && (batch_full || seq_limit_reached)
+
+        seq_id = prompts_in_batch.size
+        prompts_in_batch << seq_id
+
+        tokens.each_with_index do |token_id, pos|
+          idx = current_batch_token_count
+
+          batch[:token].put_int32(idx * FFI.type_size(:int32), token_id)
+          batch[:pos].put_int32(idx * FFI.type_size(:int32), pos)
+          batch[:n_seq_id].put_int32(idx * FFI.type_size(:int32), 1)
+          batch[:seq_id].get_pointer(idx * FFI::Pointer.size).put_int32(0, seq_id)
+          batch[:logits].put_int8(idx, pos == tokens.size - 1 ? 1 : 0)
+
+          current_batch_token_count += 1
+        end
+      end
+
+      process_batch.call
+
+      Cpp.llama_batch_free(batch)
+
+      is_array ? all_embeddings : all_embeddings[0]
+    end
+
+    def embeddings?
+      @embeddings
+    end
+
+    def close
+      Cpp.llama_free(@pointer)
+    end
+
+    def norm(vec)
+      Math.sqrt(vec.sum { |x| x**2 })
+    end
+
+    def normalize_embedding(vec)
+      n = norm(vec)
+
+      return vec if n.zero?
+
+      vec.map { |x| x / n }
+    end
+  end
+end