rllama 1.0.3-aarch64-linux-gnu → 1.2.0-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f19cb41b6197028ad125265d82dc01ca449cc6c48b5179433aefa938f139662b
-  data.tar.gz: 6ae54e060e85ce6abbfc8a22b8e5f6f66180685403516485757147eb3978e542
+  metadata.gz: 99e3ffe956c68e991661b218d27d0ed735efb0a36b6498d288d7dc4dd53fadef
+  data.tar.gz: 29cb1e766527527c2d53ad6f7d0dbbdb612c342b541cbb2a9b90c6becde859ad
 SHA512:
-  metadata.gz: dd391eaaf38dbf33bb72eb561200c9893fa7645d09948c971763ef8b8a89a1d82328d527c5fa256ae7af42f70b45edff4b2121ba45b65c3b90107aaab91523d5
-  data.tar.gz: 84f9fee026a509fe6e6c149d6d20e72e00bfbdee94e50a999b7dee8ce15d7b56f1dc4015860ce5c2e3aa8e9a643dab89e2826bef1dd1a2c98d24bcf819a27a09
+  metadata.gz: a76c30d46bd3cd39aff003fb5fcd097d13f03f0c507d23de4da0376fca057190258bf32f353e33610a3aaa2068145e97319bede5800788a197468714e2bfd44b
+  data.tar.gz: 469fc2dca9c5dac81ae290286af5ab4a07b0edf4e3488394cb3b8122218996a69528f44561226b871313ec1c430c72c4ab1a06135b27f472c65d6669f2e3ed72

data/LICENSE

@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2026 DocuSeal, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -2,7 +2,7 @@
 
 # Rllama
 
-Ruby bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) to run open-source language models locally. Run models like GPT-OSS, Qwen 3, Gemma 3, Llama 3, and many others directly in your Ruby application code.
+Ruby bindings for [llama.cpp](https://github.com/ggml-org/llama.cpp) to run open-source language models locally. Run models like Gemma 4, Qwen 3.5, GLM 4.7, Nemotron, LFM2, Llama 3, and many others directly in your Ruby application code.
 
 ## Installation
 
@@ -36,11 +36,14 @@ When you run `rllama` without arguments, it will display:
 
 - **Downloaded models**: Any models you've already downloaded to `~/.rllama/models/`
 - **Popular models**: A curated list of popular models available for download, including:
-  - Gemma 3 1B
+  - Gemma 4 E4B / Gemma 4 26B-A4B
+  - Nemotron 3 Nano 4B
+  - Qwen 3.5 35B-A3B
+  - LFM2 24B-A2B
+  - GLM 4.7 Flash
+  - GPT-OSS 20B
   - Llama 3.2 3B
   - Phi-4
-  - Qwen3 30B
-  - GPT-OSS
 
 Simply enter the number of the model you want to use. If you select a model that hasn't been downloaded yet, it will be automatically downloaded from Hugging Face.
 
@@ -84,7 +87,14 @@ model.close
 
 #### Generation parameters
 
-Adjust the generation with parameters:
+By default the sampling parameters are taken from the model's own
+`general.sampling.*` GGUF metadata when present (falling back to llama.cpp's
+standard defaults: `temperature: 0.8`, `top_k: 40`, `top_p: 0.95`,
+`min_p: 0.05`). You can inspect the resolved values with
+`model.sampling_defaults`, or read any metadata key directly with
+`model.meta('general.sampling.temp')`.
+
+Pass any parameter explicitly to override the model default:
 
 ```ruby
 result = model.generate(
@@ -97,6 +107,9 @@ result = model.generate(
 )
 ```
 
+The parameters actually used are reported back in `result.stats`
+(`temperature`, `top_k`, `top_p`, `min_p`, `seed`).
+
 #### Streaming generation
 
 Stream generated text token-by-token:
@@ -132,6 +145,95 @@ result = model.generate([
 puts result.text
 ```
 
+#### Tool definitions
+
+Pass tool definitions to include them in the model's prompt for function calling:
+
+```ruby
+tools = [
+  {
+    type: 'function',
+    function: {
+      name: 'get_weather',
+      description: 'Get the current weather for a location',
+      parameters: {
+        type: 'object',
+        properties: {
+          location: { type: 'string', description: 'City name' }
+        },
+        required: ['location']
+      }
+    }
+  }
+]
+
+result = model.generate('What is the weather in Paris?', tools:)
+puts result.text
+```
+
+#### Tool call results
+
+Messages are passed to the model's chat template in the OpenAI format, so tool
+calls and their results are wired through the standard message schema: the
+assistant message carries a structured `tool_calls:` array, and the tool output
+is sent back as a `role: 'tool'` message.
+
+Pass `tools:` (and optionally `system:`) once when creating the context — they
+are remembered for all subsequent turns. When the model emits a tool call, it
+is parsed into `result.tool_calls` and stored in the conversation history in
+structured form automatically. Execute the calls and send each result back as
+a `role: 'tool'` message:
+
+```ruby
+context = model.init_context(tools:)
+
+result = context.generate('What is the weather in Paris?')
+result.tool_calls
+# => [{ name: 'get_weather', arguments: { 'location' => 'Paris' }, id: 'call_1' }]
+
+result.tool_calls.each do |call|
+  weather = get_weather(call[:arguments]['location'])
+
+  result = context.generate(
+    { role: 'tool', tool_call_id: call[:id], name: call[:name], content: weather.to_json }
+  )
+end
+
+puts result.text
+# => "The weather in Paris right now is 21°C."
+```
+
+The tool call markup is parsed by llama.cpp's chat parser, which derives the
+format automatically from the model's own chat template — any model with a
+tool-capable template is supported. If no tool call is detected,
+`result.tool_calls` is empty and `result.text` contains the raw output.
+
+The context detects the edited history and automatically rolls the KV cache
+back to the common prefix, so only the changed suffix is re-processed.
+
+### Reasoning
+
+For models with reasoning support, pass `reasoning: true` to
+enable the model's reasoning phase. The reasoning block is parsed out of the
+output: `result.text` contains only the final answer and the reasoning is
+available separately:
+
+```ruby
+result = model.generate('Why is the sky blue?', reasoning: true)
+
+result.reasoning
+# => "Thinking Process:\n\n1. Analyze the request..."
+
+result.text
+# => "The sky appears blue because..."
+```
+
+Like `tools:`, the flag can also be set once per context via
+`model.init_context(reasoning: true)`. It defaults to off; `result.reasoning`
+is `nil` when reasoning is disabled or the model produced no reasoning block.
+Note that when streaming with a block, the raw reasoning tokens are yielded
+as they are generated.
+
 ### Chat
 
 For ongoing conversations, use a context object that maintains the conversation history:

data/bin/rllama

@@ -1,8 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'bundler/setup'
 require 'rllama'
-require 'rllama/cli'
 
 Rllama::Cli.start(ARGV)

data/lib/rllama/aarch64-linux/libggml-base.so

@@ -0,0 +1 @@
+libggml-base.so.0

data/lib/rllama/aarch64-linux/libggml-base.so.0

@@ -0,0 +1 @@
+libggml-base.so.0.17.0

data/lib/rllama/aarch64-linux/libggml.so

@@ -0,0 +1 @@
+libggml.so.0

data/lib/rllama/aarch64-linux/libggml.so.0

@@ -0,0 +1 @@
+libggml.so.0.17.0

data/lib/rllama/aarch64-linux/libllama-common.so

@@ -0,0 +1 @@
+libllama-common.so.0

data/lib/rllama/aarch64-linux/libllama-common.so.0

@@ -0,0 +1 @@
+libllama-common.so.0.0.10067

data/lib/rllama/aarch64-linux/libllama.so

@@ -0,0 +1 @@
+libllama.so.0

data/lib/rllama/aarch64-linux/libllama.so.0

@@ -0,0 +1 @@
+libllama.so.0.0.10067

data/lib/rllama/cli.rb

@@ -5,13 +5,19 @@ require 'readline'
 module Rllama
   class Cli
     POPULAR_MODELS = [
-      { path: 'lmstudio-community/gemma-3-1B-it-QAT-GGUF/gemma-3-1B-it-QAT-Q4_0.gguf', size: 720_425_472 },
-      { path: 'lmstudio-community/gpt-oss-20b-GGUF/gpt-oss-20b-MXFP4.gguf', size: 12_109_565_632 },
-      { path: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q4_K_M.gguf', size: 2_019_377_696 },
-      { path: 'unsloth/Qwen3-30B-A3B-GGUF/Qwen3-30B-A3B-Q3_K_S.gguf', size: 13_292_468_800 },
+      { path: 'lmstudio-community/gemma-4-E4B-it-GGUF/gemma-4-E4B-it-Q4_K_M.gguf', size: 5_335_285_280 },
+      { path: 'lmstudio-community/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-Q4_K_M.gguf', size: 16_796_010_624 },
+      { path: 'unsloth/NVIDIA-Nemotron-3-Nano-4B-GGUF/NVIDIA-Nemotron-3-Nano-4B-Q4_K_M.gguf', size: 2_900_295_712 },
+      { path: 'unsloth/Qwen3.5-35B-A3B-GGUF/Qwen3.5-35B-A3B-Q4_K_M.gguf', size: 22_016_023_168 },
+      { path: 'lmstudio-community/LFM2-24B-A2B-GGUF/LFM2-24B-A2B-Q4_K_M.gguf', size: 14_415_473_952 },
+      { path: 'unsloth/GLM-4.7-Flash-GGUF/GLM-4.7-Flash-Q4_K_M.gguf', size: 18_312_339_808 },
       { path: 'inclusionAI/Ling-mini-2.0-GGUF/Ling-mini-2.0-Q4_K_M.gguf', size: 9_911_575_072 },
+      { path: 'lmstudio-community/gpt-oss-20b-GGUF/gpt-oss-20b-MXFP4.gguf', size: 12_109_565_632 },
       { path: 'unsloth/gemma-3n-E4B-it-GGUF/gemma-3n-E4B-it-Q4_K_S.gguf', size: 4_404_697_216 },
-      { path: 'microsoft/phi-4-gguf/phi-4-Q4_K_S.gguf', size: 8_440_762_560 }
+      { path: 'unsloth/Qwen3-30B-A3B-GGUF/Qwen3-30B-A3B-Q3_K_S.gguf', size: 13_292_468_800 },
+      { path: 'lmstudio-community/gemma-3-1B-it-QAT-GGUF/gemma-3-1B-it-QAT-Q4_0.gguf', size: 720_425_472 },
+      { path: 'microsoft/phi-4-gguf/phi-4-Q4_K_S.gguf', size: 8_440_762_560 },
+      { path: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q4_K_M.gguf', size: 2_019_377_696 }
     ].freeze
 
     COLOR_CODES = {

data/lib/rllama/common.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'ffi'
+require 'json'
+
+module Rllama
+  module Common
+    extend FFI::Library
+
+    REASONING_FORMAT_AUTO = 1
+
+    lib_file =
+      case FFI::Platform::OS
+      when 'darwin'
+        'libllama_common.dylib'
+      when 'windows', 'mingw32'
+        'llama_common.dll'
+      else
+        'libllama_common.so'
+      end
+
+    ffi_lib File.join(Cpp::PLATFORM_DIR, lib_file)
+
+    attach_function :rlc_free, [:pointer], :void
+    attach_function :rlc_chat_templates_init, %i[pointer string string string pointer], :pointer
+    attach_function :rlc_chat_templates_free, [:pointer], :void
+    attach_function :rlc_chat_apply, %i[pointer string pointer], :pointer
+    attach_function :rlc_chat_parse, %i[string bool string pointer], :pointer
+    attach_function :rlc_sampler_init, %i[pointer string pointer], :pointer
+    attach_function :rlc_sampler_free, [:pointer], :void
+    attach_function :rlc_sampler_sample, %i[pointer pointer int32], :int32
+    attach_function :rlc_sampler_accept, %i[pointer int32 bool], :void
+    attach_function :rlc_sampler_reset, [:pointer], :void
+    attach_function :rlc_sampler_print, [:pointer], :pointer
+
+    module_function
+
+    def chat_templates_init(model, source, bos_token, eos_token)
+      with_err do |err|
+        handle = rlc_chat_templates_init(model, source, bos_token, eos_token, err)
+
+        raise Error, "Failed to parse chat template: #{read_err(err)}" if handle.null?
+
+        handle
+      end
+    end
+
+    def chat_templates_free(handle)
+      rlc_chat_templates_free(handle)
+    end
+
+    def chat_apply(handle, messages:, tools: nil, add_generation_prompt: true, enable_thinking: false,
+                   add_bos: false, add_eos: false)
+      inputs = { messages:, tools:, add_generation_prompt:, enable_thinking:, add_bos:, add_eos:,
+                 reasoning_format: enable_thinking ? REASONING_FORMAT_AUTO : 0 }.to_json
+
+      with_err do |err|
+        result = rlc_chat_apply(handle, inputs, err)
+
+        raise Error, "Failed to apply chat template: #{read_err(err)}" if result.null?
+
+        JSON.parse(take(result))
+      end
+    end
+
+    def chat_parse(text, params, is_partial: false, reasoning: false)
+      parse_params = params.slice('format', 'parser', 'generation_prompt')
+      parse_params['reasoning_format'] = REASONING_FORMAT_AUTO if reasoning
+      params_json = parse_params.to_json
+
+      with_err do |err|
+        result = rlc_chat_parse(text, is_partial, params_json, err)
+
+        raise Error, "Failed to parse output: #{read_err(err)}" if result.null?
+
+        JSON.parse(take(result))
+      end
+    end
+
+    def sampler_init(model, params)
+      with_err do |err|
+        handle = rlc_sampler_init(model, params.to_json, err)
+
+        raise Error, "Failed to initialize sampler: #{read_err(err)}" if handle.null?
+
+        handle
+      end
+    end
+
+    def sampler_free(handle)
+      rlc_sampler_free(handle)
+    end
+
+    def sampler_sample(handle, ctx, idx = -1)
+      rlc_sampler_sample(handle, ctx, idx)
+    end
+
+    def sampler_accept(handle, token, accept_grammar: true)
+      rlc_sampler_accept(handle, token, accept_grammar)
+    end
+
+    def sampler_reset(handle)
+      rlc_sampler_reset(handle)
+    end
+
+    def sampler_print(handle)
+      take(rlc_sampler_print(handle))
+    end
+
+    def with_err
+      yield FFI::MemoryPointer.new(:pointer)
+    end
+
+    def take(ptr)
+      str = ptr.read_string
+
+      rlc_free(ptr)
+
+      str
+    end
+
+    def read_err(err)
+      msg_ptr = err.read_pointer
+
+      return 'unknown error' if msg_ptr.null?
+
+      msg = msg_ptr.read_string
+
+      rlc_free(msg_ptr)
+
+      msg
+    end
+  end
+end