red-candle 1.3.0 → 1.4.0

data/ext/candle/src/ruby/tokenizer.rs

@@ -1,4 +1,4 @@
-use magnus::{class, function, method, prelude::*, Error, Module, RArray, RHash, RModule, TryConvert};
+use magnus::{function, method, prelude::*, Error, Module, RArray, RHash, RModule, Ruby, TryConvert};
 use crate::tokenizer::{TokenizerWrapper as InnerTokenizer, loader::TokenizerLoader};
 use crate::ruby::Result;
 
@@ -9,136 +9,151 @@ pub struct Tokenizer(pub InnerTokenizer);
 impl Tokenizer {
     /// Create a new tokenizer from a file path
     pub fn from_file(path: String) -> Result<Self> {
+        let ruby = Ruby::get().unwrap();
         let tokenizer = TokenizerLoader::from_file(&path)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))?;
         Ok(Self(InnerTokenizer::new(tokenizer)))
     }
 
     /// Create a new tokenizer from HuggingFace model ID
     pub fn from_pretrained(model_id: String) -> Result<Self> {
+        let ruby = Ruby::get().unwrap();
+        let runtime_error = ruby.exception_runtime_error();
         // Use tokio runtime for async operations
         let rt = tokio::runtime::Runtime::new()
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), format!("Failed to create runtime: {}", e)))?;
-        
+            .map_err(|e| Error::new(runtime_error, format!("Failed to create runtime: {}", e)))?;
+
         let tokenizer = rt.block_on(async {
             TokenizerLoader::from_hf_hub(&model_id, None).await
         })
-        .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
+        .map_err(|e| Error::new(runtime_error, e.to_string()))?;
+
         Ok(Self(InnerTokenizer::new(tokenizer)))
     }
 
     /// Encode text into token IDs
     pub fn encode(&self, text: String, add_special_tokens: Option<bool>) -> Result<RArray> {
+        let ruby = Ruby::get().unwrap();
         let add_special = add_special_tokens.unwrap_or(true);
         let token_ids = self.0.encode(&text, add_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
-        Ok(RArray::from_vec(token_ids.into_iter().map(|id| id as i64).collect()))
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))?;
+
+        Ok(ruby.ary_from_vec(token_ids.into_iter().map(|id| id as i64).collect()))
     }
-    
+
     /// Encode text into token strings (words/subwords)
     pub fn encode_to_tokens(&self, text: String, add_special_tokens: Option<bool>) -> Result<RArray> {
+        let ruby = Ruby::get().unwrap();
+        let runtime_error = ruby.exception_runtime_error();
         let add_special = add_special_tokens.unwrap_or(true);
         let token_ids = self.0.encode(&text, add_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
+            .map_err(|e| Error::new(runtime_error, e.to_string()))?;
+
         let mut tokens = Vec::new();
         for id in token_ids {
             let token = self.0.token_to_piece(id)
-                .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
+                .map_err(|e| Error::new(runtime_error, e.to_string()))?;
             tokens.push(token);
         }
-        
-        Ok(RArray::from_vec(tokens))
+
+        Ok(ruby.ary_from_vec(tokens))
     }
 
     /// Encode multiple texts in batch
     pub fn encode_batch(&self, texts: RArray, add_special_tokens: Option<bool>) -> Result<RArray> {
+        let ruby = Ruby::get().unwrap();
+        let runtime_error = ruby.exception_runtime_error();
         let texts: Vec<String> = texts.to_vec()?;
         let add_special = add_special_tokens.unwrap_or(true);
-        
+
         let token_ids_batch = self.0.encode_batch(texts, add_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
-        let result = RArray::new();
+            .map_err(|e| Error::new(runtime_error, e.to_string()))?;
+
+        let result = ruby.ary_new();
         for token_ids in token_ids_batch {
-            result.push(RArray::from_vec(token_ids.into_iter().map(|id| id as i64).collect()))?;
+            result.push(ruby.ary_from_vec(token_ids.into_iter().map(|id| id as i64).collect()))?;
         }
-        
+
         Ok(result)
     }
-    
+
     /// Encode multiple texts in batch, returning token strings
     pub fn encode_batch_to_tokens(&self, texts: RArray, add_special_tokens: Option<bool>) -> Result<RArray> {
+        let ruby = Ruby::get().unwrap();
+        let runtime_error = ruby.exception_runtime_error();
         let texts: Vec<String> = texts.to_vec()?;
         let add_special = add_special_tokens.unwrap_or(true);
-        
+
         let token_ids_batch = self.0.encode_batch(texts, add_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
-        let result = RArray::new();
+            .map_err(|e| Error::new(runtime_error, e.to_string()))?;
+
+        let result = ruby.ary_new();
         for token_ids in token_ids_batch {
             let mut tokens = Vec::new();
             for id in token_ids {
                 let token = self.0.token_to_piece(id)
-                    .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
+                    .map_err(|e| Error::new(runtime_error, e.to_string()))?;
                 tokens.push(token);
             }
-            result.push(RArray::from_vec(tokens))?;
+            result.push(ruby.ary_from_vec(tokens))?;
         }
-        
+
         Ok(result)
     }
 
     /// Encode text and return both token IDs and token strings
     pub fn encode_with_tokens(&self, text: String, add_special_tokens: Option<bool>) -> Result<RHash> {
+        let ruby = Ruby::get().unwrap();
+        let runtime_error = ruby.exception_runtime_error();
         let add_special = add_special_tokens.unwrap_or(true);
         let token_ids = self.0.encode(&text, add_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
-        
+            .map_err(|e| Error::new(runtime_error, e.to_string()))?;
+
         let mut tokens = Vec::new();
         for &id in &token_ids {
             let token = self.0.token_to_piece(id)
-                .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))?;
+                .map_err(|e| Error::new(runtime_error, e.to_string()))?;
             tokens.push(token);
         }
-        
-        let hash = RHash::new();
-        hash.aset(magnus::Symbol::new("ids"), RArray::from_vec(token_ids.into_iter().map(|id| id as i64).collect()))?;
-        hash.aset(magnus::Symbol::new("tokens"), RArray::from_vec(tokens))?;
-        
+
+        let hash = ruby.hash_new();
+        hash.aset(ruby.to_symbol("ids"), ruby.ary_from_vec(token_ids.into_iter().map(|id| id as i64).collect()))?;
+        hash.aset(ruby.to_symbol("tokens"), ruby.ary_from_vec(tokens))?;
+
         Ok(hash)
     }
-    
+
     /// Decode token IDs back to text
     pub fn decode(&self, token_ids: RArray, skip_special_tokens: Option<bool>) -> Result<String> {
+        let ruby = Ruby::get().unwrap();
         let token_ids: Vec<i64> = token_ids.to_vec()?;
         let token_ids: Vec<u32> = token_ids.into_iter()
             .map(|id| id as u32)
             .collect();
         let skip_special = skip_special_tokens.unwrap_or(true);
-        
+
         self.0.decode(&token_ids, skip_special)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))
     }
 
     /// Get the string representation of a single token ID
     pub fn id_to_token(&self, token_id: i64) -> Result<String> {
+        let ruby = Ruby::get().unwrap();
         self.0.token_to_piece(token_id as u32)
-            .map_err(|e| Error::new(magnus::exception::runtime_error(), e.to_string()))
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))
     }
 
     /// Get the vocabulary as a hash of token string to ID
     pub fn get_vocab(&self, with_added_tokens: Option<bool>) -> Result<RHash> {
+        let ruby = Ruby::get().unwrap();
         let with_added = with_added_tokens.unwrap_or(true);
         let vocab = self.0.inner().get_vocab(with_added);
-        
-        let hash = RHash::new();
+
+        let hash = ruby.hash_new();
         for (token, id) in vocab {
             hash.aset(token, id as i64)?;
         }
-        
+
         Ok(hash)
     }
 
@@ -151,23 +166,24 @@ impl Tokenizer {
     /// Enable padding - returns a new tokenizer with padding enabled
     pub fn with_padding(&self, kwargs: RHash) -> Result<Self> {
         use tokenizers::{PaddingParams, PaddingStrategy, PaddingDirection};
-        
+        let ruby = Ruby::get().unwrap();
+
         let mut params = PaddingParams::default();
-        
+
         // Extract parameters from kwargs
-        if let Some(length) = kwargs.get(magnus::Symbol::new("length")) {
+        if let Some(length) = kwargs.get(ruby.to_symbol("length")) {
             if let Ok(len) = usize::try_convert(length) {
                 params.strategy = PaddingStrategy::Fixed(len);
             }
         }
-        
-        if let Some(max_length) = kwargs.get(magnus::Symbol::new("max_length")) {
+
+        if let Some(max_length) = kwargs.get(ruby.to_symbol("max_length")) {
             if let Ok(_) = usize::try_convert(max_length) {
                 params.strategy = PaddingStrategy::BatchLongest;
             }
         }
-        
-        if let Some(direction) = kwargs.get(magnus::Symbol::new("direction")) {
+
+        if let Some(direction) = kwargs.get(ruby.to_symbol("direction")) {
             if let Ok(dir) = String::try_convert(direction) {
                 params.direction = match dir.as_str() {
                     "right" => PaddingDirection::Right,
@@ -176,19 +192,19 @@ impl Tokenizer {
                 };
             }
         }
-        
-        if let Some(pad_id) = kwargs.get(magnus::Symbol::new("pad_id")) {
+
+        if let Some(pad_id) = kwargs.get(ruby.to_symbol("pad_id")) {
             if let Ok(id) = u32::try_convert(pad_id) {
                 params.pad_id = id;
             }
         }
-        
-        if let Some(pad_token) = kwargs.get(magnus::Symbol::new("pad_token")) {
+
+        if let Some(pad_token) = kwargs.get(ruby.to_symbol("pad_token")) {
             if let Ok(token) = String::try_convert(pad_token) {
                 params.pad_token = token;
             }
         }
-        
+
         let mut new_tokenizer = self.0.clone();
         let _ = new_tokenizer.inner_mut().with_padding(Some(params));
         Ok(Self(new_tokenizer))
@@ -197,14 +213,14 @@ impl Tokenizer {
     /// Enable truncation - returns a new tokenizer with truncation enabled
     pub fn with_truncation(&self, max_length: usize) -> Result<Self> {
         use tokenizers::{TruncationParams, TruncationStrategy, TruncationDirection};
-        
+
         let params = TruncationParams {
             max_length,
             strategy: TruncationStrategy::LongestFirst,
             stride: 0,
             direction: TruncationDirection::Right,
         };
-        
+
         let mut new_tokenizer = self.0.clone();
         let _ = new_tokenizer.inner_mut().with_truncation(Some(params));
         Ok(Self(new_tokenizer))
@@ -212,8 +228,9 @@ impl Tokenizer {
 
     /// Get special tokens information
     pub fn get_special_tokens(&self) -> Result<RHash> {
-        let hash = RHash::new();
-        
+        let ruby = Ruby::get().unwrap();
+        let hash = ruby.hash_new();
+
         // Common special tokens
         let special_tokens = vec![
             ("[CLS]", "cls_token"),
@@ -224,50 +241,49 @@ impl Tokenizer {
             ("<s>", "bos_token"),
             ("</s>", "eos_token"),
         ];
-        
+
         let vocab = self.0.inner().get_vocab(true);
-        
+
         for (token, name) in special_tokens {
             if let Some(id) = vocab.get(token) {
                 hash.aset(name, *id as i64)?;
             }
         }
-        
+
         Ok(hash)
     }
 
     /// Get tokenizer options as a hash
     pub fn options(&self) -> Result<RHash> {
-        let hash = RHash::new();
-        
+        let ruby = Ruby::get().unwrap();
+        let hash = ruby.hash_new();
+
         // Get vocab size
         hash.aset("vocab_size", self.vocab_size(Some(true)))?;
         hash.aset("vocab_size_base", self.vocab_size(Some(false)))?;
-        
+
         // Get special tokens info
         let special_tokens = self.get_special_tokens()?;
         hash.aset("special_tokens", special_tokens)?;
-        
+
         // Get padding/truncation info if available
         let inner_tokenizer = self.0.inner();
-        
+
         // Check if padding is enabled
         if let Some(_padding) = inner_tokenizer.get_padding() {
-            let padding_info = RHash::new();
+            let padding_info = ruby.hash_new();
             padding_info.aset("enabled", true)?;
-            // Note: We can't easily extract all padding params from the tokenizers library
-            // but we can indicate it's enabled
             hash.aset("padding", padding_info)?;
         }
-        
-        // Check if truncation is enabled  
+
+        // Check if truncation is enabled
         if let Some(truncation) = inner_tokenizer.get_truncation() {
-            let truncation_info = RHash::new();
+            let truncation_info = ruby.hash_new();
             truncation_info.aset("enabled", true)?;
             truncation_info.aset("max_length", truncation.max_length)?;
             hash.aset("truncation", truncation_info)?;
         }
-        
+
         Ok(hash)
     }
 
@@ -278,13 +294,13 @@ impl Tokenizer {
             .ok()
             .map(|h| h.len())
             .unwrap_or(0);
-        
+
         let mut parts = vec![format!("#<Candle::Tokenizer vocab_size={}", vocab_size)];
-        
+
         if special_tokens > 0 {
             parts.push(format!("special_tokens={}", special_tokens));
         }
-        
+
         // Check for padding/truncation
         let inner_tokenizer = self.0.inner();
         if inner_tokenizer.get_padding().is_some() {
@@ -293,18 +309,19 @@ impl Tokenizer {
         if let Some(truncation) = inner_tokenizer.get_truncation() {
             parts.push(format!("truncation={}", truncation.max_length));
         }
-        
+
         parts.join(" ") + ">"
     }
 }
 
 pub fn init(rb_candle: RModule) -> Result<()> {
-    let tokenizer_class = rb_candle.define_class("Tokenizer", class::object())?;
-    
+    let ruby = Ruby::get().unwrap();
+    let tokenizer_class = rb_candle.define_class("Tokenizer", ruby.class_object())?;
+
     // Class methods
     tokenizer_class.define_singleton_method("from_file", function!(Tokenizer::from_file, 1))?;
     tokenizer_class.define_singleton_method("from_pretrained", function!(Tokenizer::from_pretrained, 1))?;
-    
+
     // Instance methods
     tokenizer_class.define_method("encode", method!(Tokenizer::encode, 2))?;
     tokenizer_class.define_method("encode_to_tokens", method!(Tokenizer::encode_to_tokens, 2))?;
@@ -321,6 +338,6 @@ pub fn init(rb_candle: RModule) -> Result<()> {
     tokenizer_class.define_method("options", method!(Tokenizer::options, 0))?;
     tokenizer_class.define_method("inspect", method!(Tokenizer::inspect, 0))?;
     tokenizer_class.define_method("to_s", method!(Tokenizer::inspect, 0))?;
-    
+
     Ok(())
-}
+}

data/lib/candle/llm.rb

@@ -32,16 +32,90 @@ module Candle
       end
     end
     # Create a structured constraint from a JSON schema
+    # Uses the model's vocabulary with proper byte encoding handling
     def constraint_from_schema(schema)
       schema_str = schema.is_a?(String) ? schema : JSON.generate(schema)
-      StructuredConstraint.from_schema(schema_str, tokenizer)
+
+      # Extract the tokenizer source model ID for proper vocabulary loading
+      tokenizer_model = tokenizer_source_model
+      if tokenizer_model
+        begin
+          StructuredConstraint.from_schema_with_model(schema_str, tokenizer_model)
+        rescue RuntimeError => e
+          # Fall back to legacy method if from_pretrained fails
+          # (e.g., tokenizer doesn't have EOS token in expected format)
+          if e.message.include?("UnsupportedTokenizer")
+            StructuredConstraint.from_schema(schema_str, tokenizer)
+          else
+            raise
+          end
+        end
+      else
+        # Fall back to legacy method if we can't determine the model
+        StructuredConstraint.from_schema(schema_str, tokenizer)
+      end
     end
-    
+
     # Create a structured constraint from a regex pattern
+    # Uses the model's vocabulary with proper byte encoding handling
     def constraint_from_regex(pattern)
       pattern_str = pattern.is_a?(Regexp) ? pattern.source : pattern.to_s
-      StructuredConstraint.from_regex(pattern_str, tokenizer)
+
+      # Extract the tokenizer source model ID for proper vocabulary loading
+      tokenizer_model = tokenizer_source_model
+      if tokenizer_model
+        begin
+          StructuredConstraint.from_regex_with_model(pattern_str, tokenizer_model)
+        rescue RuntimeError => e
+          # Fall back to legacy method if from_pretrained fails
+          if e.message.include?("UnsupportedTokenizer")
+            StructuredConstraint.from_regex(pattern_str, tokenizer)
+          else
+            raise
+          end
+        end
+      else
+        # Fall back to legacy method if we can't determine the model
+        StructuredConstraint.from_regex(pattern_str, tokenizer)
+      end
     end
+
+    private
+
+    # Get the model ID to use for vocabulary loading
+    # This handles GGUF models by extracting the tokenizer source
+    def tokenizer_source_model
+      opts = options rescue {}
+
+      # For GGUF models, use the tokenizer source if available
+      if opts["tokenizer_source"]
+        return opts["tokenizer_source"]
+      end
+
+      # For regular models, use the base model ID
+      if opts["base_model"]
+        return opts["base_model"]
+      end
+
+      # Try model_id but strip GGUF parts
+      model = opts["model_id"] || (model_id rescue nil)
+      return nil unless model
+
+      # Remove GGUF file suffix if present
+      if model.include?("@")
+        model = model.split("@").first
+      end
+
+      # For GGUF repos, try to guess the tokenizer source
+      if model.downcase.include?("gguf")
+        guessed = self.class.guess_tokenizer(model)
+        return guessed if guessed && guessed != model
+      end
+
+      model
+    end
+
+    public
     
     # Generate with regex constraint
     def generate_regex(prompt, pattern:, stop_on_match: true, **options)

data/lib/candle/version.rb

@@ -1,5 +1,5 @@
 # :nocov:
 module Candle
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end
 # :nocov:

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-candle
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Christopher Petersen
 - kojix2
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-09-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -25,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -225,7 +252,6 @@ homepage: https://github.com/scientist-labs/red-candle
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -241,8 +267,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '3.3'
 requirements:
 - Rust >= 1.85
-rubygems_version: 3.3.3
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Ruby gem for running state-of-the-art language models locally. Access LLMs,
   embeddings, rerankers, and NER models directly from Ruby using Rust-powered Candle