rllama 1.0.1-aarch64-linux-musl

data/lib/rllama/loader.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'net/http'
+require 'fileutils'
+
+module Rllama
+  module Loader
+    HUGGINGFACE_BASE_URL = 'https://huggingface.co'
+    DEFAULT_DIR = File.join(Dir.home, '.rllama')
+
+    UNITS = %w[B KB MB GB TB].freeze
+
+    module_function
+
+    def resolve(path_or_name, dir: nil)
+      dir ||= DEFAULT_DIR
+
+      dir = File.join(dir, 'models')
+
+      return path_or_name if local_file?(path_or_name)
+
+      if url?(path_or_name)
+        download_from_url(path_or_name, dir)
+      elsif huggingface_path?(path_or_name)
+        download_from_huggingface(path_or_name, dir)
+      else
+        raise Error, "Invalid model path or name: #{path_or_name}"
+      end
+    end
+
+    def local_file?(path)
+      File.exist?(path)
+    end
+
+    def url?(path)
+      uri = URI.parse(path)
+
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def huggingface_path?(path)
+      return false if path.start_with?('/') || path.include?('://')
+
+      parts = path.split('/')
+
+      parts.length >= 3 && parts.last.end_with?('.gguf')
+    end
+
+    def download_from_huggingface(hf_path, dir)
+      parts = hf_path.split('/')
+
+      raise Error, "Invalid HuggingFace path: #{hf_path}" if parts.length < 3
+
+      org = parts[0]
+      repo = parts[1]
+      file_path = parts[2..].join('/')
+
+      url = "#{HUGGINGFACE_BASE_URL}/#{org}/#{repo}/resolve/main/#{file_path}"
+
+      local_path = File.join(dir, org, repo, file_path)
+
+      return local_path if File.exist?(local_path)
+
+      puts "Destination: #{local_path}"
+
+      download_file(url, local_path, "HuggingFace model: #{hf_path}")
+    end
+
+    def download_from_url(url, dir)
+      uri = URI.parse(url)
+
+      filename = File.basename(uri.path)
+
+      local_path = File.join(dir, filename)
+
+      return local_path if File.exist?(local_path)
+
+      puts "Destination: #{local_path}"
+
+      download_file(url, local_path, "URL: #{url}")
+    end
+
+    def download_file(url, local_path, description)
+      FileUtils.mkdir_p(File.dirname(local_path))
+
+      temp_path = File.join(File.dirname(local_path), "~#{File.basename(local_path)}")
+
+      existing_size = File.exist?(temp_path) ? File.size(temp_path) : 0
+
+      uri = URI.parse(url)
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        request = Net::HTTP::Get.new(uri.request_uri)
+
+        request['Range'] = "bytes=#{existing_size}-" if existing_size.positive?
+
+        http.request(request) do |response|
+          case response
+          when Net::HTTPSuccess, Net::HTTPPartialContent
+            if response['Content-Range']
+              total_size = response['Content-Range'].split('/').last.to_i
+            else
+              total_size = response['content-length'].to_i
+
+              if existing_size.positive? && response.code == '200'
+                puts "\nServer doesn't support resume, starting from beginning..."
+
+                existing_size = 0
+
+                FileUtils.rm_f(temp_path)
+              end
+            end
+
+            downloaded = existing_size
+            file_mode = existing_size.positive? ? 'ab' : 'wb'
+
+            File.open(temp_path, file_mode) do |file|
+              response.read_body do |chunk|
+                file.write(chunk)
+                downloaded += chunk.size
+
+                if total_size.positive?
+                  progress = (downloaded.to_f / total_size * 100).round
+                  total_str = format_bytes(total_size)
+                  downloaded_str = format_bytes(downloaded)
+                  padding = total_str.length
+                  formatted_downloaded = format("%#{padding}s", downloaded_str)
+                  print format("\rProgress: %<progress>6d%% (%<downloaded>s / %<total>s)",
+                               progress: progress, downloaded: formatted_downloaded, total: total_str)
+                else
+                  print "\rDownloaded: #{format_bytes(downloaded)}"
+                end
+              end
+            end
+
+            unless verify_download(temp_path, total_size)
+              FileUtils.rm_f(temp_path)
+
+              raise Error, 'Download verification failed - file size mismatch'
+            end
+
+            File.rename(temp_path, local_path)
+
+            puts
+          when Net::HTTPRedirection
+            redirect_url = response['location']
+
+            redirect_url = URI.join(url, redirect_url).to_s unless redirect_url.start_with?('http://', 'https://')
+
+            return download_file(redirect_url, local_path, description)
+          when Net::HTTPRequestedRangeNotSatisfiable
+            if File.exist?(temp_path)
+              uri = URI.parse(url)
+
+              Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |check_http|
+                head_request = Net::HTTP::Head.new(uri.request_uri)
+                head_response = check_http.request(head_request)
+
+                if head_response.is_a?(Net::HTTPSuccess)
+                  expected_size = head_response['content-length'].to_i
+                  actual_size = File.size(temp_path)
+
+                  if expected_size.positive? && expected_size == actual_size
+                    File.rename(temp_path, local_path)
+
+                    return local_path
+                  end
+                end
+              end
+
+              File.delete(temp_path)
+
+              return download_file(url, local_path, description)
+            end
+
+            raise Error, "Range request failed: #{response.code} #{response.message}"
+          else
+            raise Error, "Failed to download model: #{response.code} #{response.message}"
+          end
+        end
+      end
+
+      local_path
+    end
+
+    def verify_download(local_path, expected_size)
+      return true if expected_size <= 0
+
+      actual_size = File.size(local_path)
+      actual_size == expected_size
+    end
+
+    def format_bytes(bytes)
+      return '0 B' if bytes.zero?
+
+      exp = (Math.log(bytes) / Math.log(1024)).floor
+
+      exp = [exp, UNITS.length - 1].min
+
+      value = bytes.to_f / (1024**exp)
+
+      if exp >= 3
+        format('%<val>.2f %<unit>s', val: value, unit: UNITS[exp])
+      else
+        format('%<val>d %<unit>s', val: value.round, unit: UNITS[exp])
+      end
+    end
+  end
+end

data/lib/rllama/model.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Rllama
+  class Model
+    DEFAULT_CONTEXT_LENGTH = 2**13
+
+    attr_reader :pointer
+
+    def initialize(path_or_name, dir: nil)
+      resolved_path = Loader.resolve(path_or_name, dir:)
+
+      model_params = Cpp.llama_model_default_params
+
+      @pointer = Cpp.llama_model_load_from_file(resolved_path, model_params)
+
+      raise Error, "Unable to load model from #{resolved_path}" if @pointer.null?
+    end
+
+    def chat_template
+      @chat_template ||= Cpp.llama_model_chat_template(@pointer, nil)
+    end
+
+    def vocab
+      @vocab ||= Cpp.llama_model_get_vocab(@pointer)
+    end
+
+    def n_embd
+      @n_embd ||= Cpp.llama_model_n_embd(@pointer)
+    end
+
+    def n_seq_max
+      @n_seq_max ||= Cpp.llama_max_parallel_sequences
+    end
+
+    def n_ctx_train
+      @n_ctx_train ||= Cpp.llama_model_n_ctx_train(@pointer)
+    end
+
+    def generate(prompt, max_tokens: DEFAULT_CONTEXT_LENGTH, temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05,
+                 seed: nil, system: nil, &block)
+      init_context(n_ctx: max_tokens) do |ctx|
+        ctx.generate(prompt, max_tokens: ctx.n_ctx,
+                             temperature:, top_k:, top_p:, seed:, system:, min_p:,
+                     &block)
+      end
+    end
+    alias message generate
+
+    def embed(prompt, normalize: true, batch_size: 512, &block)
+      inputs = prompt.is_a?(Array) ? prompt : [prompt]
+
+      tokenized_inputs = inputs.map { |text| tokenize(text, max_tokens: n_ctx_train) }
+      max_token_length = tokenized_inputs.map(&:length).max || 0
+
+      effective_batch_size = [batch_size, max_token_length].max
+      effective_ctx = [n_ctx_train, max_token_length].min
+
+      init_embedding_context(n_ctx: effective_ctx, n_batch: effective_batch_size) do |ctx|
+        inputs = prompt.is_a?(Array) ? tokenized_inputs : tokenized_inputs[0]
+
+        ctx.embed(inputs, normalize:, batch_size: effective_batch_size, &block)
+      end
+    end
+
+    def tokenize(text, max_tokens: nil)
+      size = text.bytesize + 2
+
+      tokens_ptr = FFI::MemoryPointer.new(:int32, size)
+      count = Cpp.llama_tokenize(vocab, text, text.bytesize, tokens_ptr, size, true, false)
+
+      raise Error, "Failed to tokenize text: '#{text}'" if count.negative?
+
+      tokens_ptr.read_array_of_int32([count, max_tokens].compact.min)
+    end
+
+    def close
+      Cpp.llama_model_free(@pointer)
+    end
+
+    def init_context(embeddings: false, n_ctx: DEFAULT_CONTEXT_LENGTH, n_batch: 512)
+      context = Context.new(self, embeddings:, n_ctx:, n_batch:)
+
+      if block_given?
+        result = yield context
+
+        context.close
+
+        return result
+      end
+
+      context
+    end
+
+    def init_embedding_context(n_ctx: n_ctx_train, n_batch: 512, &)
+      init_context(embeddings: true, n_ctx:, n_batch:, &)
+    end
+
+    def build_chat_template(messages)
+      raise Error, 'Model does not provide a chat template' if chat_template.nil? || chat_template.empty?
+
+      count = messages.length
+      struct_size = Cpp::LlamaChatMessage.size
+      array_ptr = FFI::MemoryPointer.new(struct_size * count)
+
+      messages.each_with_index do |m, i|
+        struct_ptr = array_ptr + (i * struct_size)
+        msg_struct = Cpp::LlamaChatMessage.new(struct_ptr)
+        msg_struct[:role] = FFI::MemoryPointer.from_string(m[:role].to_s)
+        msg_struct[:content] = FFI::MemoryPointer.from_string(m[:content].to_s)
+      end
+
+      needed = Cpp.llama_chat_apply_template(chat_template, array_ptr, count, true, nil, 0)
+
+      raise Error, 'Failed to apply chat template' if needed.negative?
+
+      buf = FFI::MemoryPointer.new(:char, needed)
+      written = Cpp.llama_chat_apply_template(chat_template, array_ptr, count, true, buf, needed)
+
+      raise Error, 'Failed to apply chat template' if written.negative?
+
+      buf.read_string(written)
+    end
+  end
+end

data/lib/rllama/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Rllama
+  VERSION = '1.0.1'
+end

data/lib/rllama.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Rllama
+  autoload :Model, 'rllama/model'
+  autoload :Loader, 'rllama/loader'
+  autoload :Context, 'rllama/context'
+  autoload :Cpp, 'rllama/cpp'
+  autoload :Cli, 'rllama/cli'
+  autoload :VERSION, 'rllama/version'
+
+  Result = Struct.new(:text, :stats, keyword_init: true)
+  Error = Class.new(StandardError)
+
+  module_function
+
+  def load_model(path_or_name, dir: nil)
+    model = Model.new(path_or_name, dir:)
+
+    if block_given?
+      begin
+        yield model
+      ensure
+        model.close
+      end
+    else
+      model
+    end
+  end
+
+  def silence_log!
+    Cpp.silence_log!
+  end
+
+  def set_log(io = $stdout)
+    Cpp.set_log(io)
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: rllama
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: aarch64-linux-musl
+authors:
+- Pete Matsyburka
+bindir: bin
+cert_chain: []
+date: 2025-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Ruby bindings for Llama.cpp to run local LLMs in Ruby applications.
+email:
+- pete@docuseal.com
+executables:
+- rllama
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/rllama
+- lib/rllama.rb
+- lib/rllama/aarch64-linux/libggml-base.so
+- lib/rllama/aarch64-linux/libggml-cpu-alderlake.so
+- lib/rllama/aarch64-linux/libggml-cpu-haswell.so
+- lib/rllama/aarch64-linux/libggml-cpu-icelake.so
+- lib/rllama/aarch64-linux/libggml-cpu-sandybridge.so
+- lib/rllama/aarch64-linux/libggml-cpu-sapphirerapids.so
+- lib/rllama/aarch64-linux/libggml-cpu-skylakex.so
+- lib/rllama/aarch64-linux/libggml-cpu-sse42.so
+- lib/rllama/aarch64-linux/libggml-cpu-x64.so
+- lib/rllama/aarch64-linux/libggml-cpu.so
+- lib/rllama/aarch64-linux/libggml-rpc.so
+- lib/rllama/aarch64-linux/libggml.so
+- lib/rllama/aarch64-linux/libllama.so
+- lib/rllama/cli.rb
+- lib/rllama/context.rb
+- lib/rllama/cpp.rb
+- lib/rllama/loader.rb
+- lib/rllama/model.rb
+- lib/rllama/version.rb
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/docusealco/rllama/issues
+  homepage_uri: https://github.com/docusealco/rllama
+  source_code_uri: https://github.com/docusealco/rllama
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Ruby bindings for Llama API
+test_files: []