red-candle 1.0.0.pre.7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bea0c9f27d5bbbc43e0c2e8fa5456b2b79511f164f7083512025538ca172077
-  data.tar.gz: b591f559c63c1bdb9fa96beaf1079186284b219e2ceba50d8e2891fa4f3096bd
+  metadata.gz: 829a937851c782dfd58b8fb724dc7b08d524d26400047e9f5fc7a5bd0de9cb4b
+  data.tar.gz: e8a9420fc310e977968aa396a47e5e5269d107eb8cb7246ca9e2f980a0a28f4d
 SHA512:
-  metadata.gz: 380cde32abf995f9766056638d4d0612acef746a84b31451c0aa2fb8cd205fc76b76b8909729a9af267ee6c549f9149253b2bfb70717385b5afe2cd2ad0b3152
-  data.tar.gz: 7acdfc4de263501be51e3f74933c528b9d723b4da93e78e3abae2454dc2746fb7696c09722fe6666084a48dff098501b2d1af503963b5d739e879bd0fcb96e43
+  metadata.gz: 020e23df61d5679612a7892bdbfc7dbcf2d28055df9fc6b9199a8e09933e74d03a0a0fb97d6ddc998f8f8e70e856d63d7eb758638e072cd492734b21681267b7
+  data.tar.gz: 5e10f888c2bd74dfdf01c97ca18f6666440edb000d524784ad0ac0676208d9485b8d708fa72d8fe73672f018c038b3526109153540ce5bbc53a81e4e30deccd1

data/Gemfile

@@ -1,12 +1,3 @@
 source "https://rubygems.org"
 
-gemspec
-
-gem "minitest"
-gem "rake"
-gem "rake-compiler"
-
-gem "yard", require: false
-gem "yard-rustdoc", require: false
-
-gem "redcarpet", "~> 3.6"
+gemspec

data/README.md

@@ -127,6 +127,8 @@ response = llm.chat(messages)
 
 ### GPU Acceleration
 
+We see an 18x speed up running LLMs under CUDA vs CPU and a >3x speed up running under Metal vs CPU. Details [here](DEVICE_SUPPORT.md#performance-considerations).
+
 ```ruby
 # CPU works for all models
 device = Candle::Device.cpu
@@ -166,9 +168,11 @@ This is particularly useful for:
 - Troubleshooting generation problems
 - Analyzing model behavior
 
-## ⚠️ Model Format Requirement: Safetensors Only
+## ⚠️ Model Format Requirements
+
+### EmbeddingModels and Rerankers: Safetensors Only
 
-Red-Candle **only supports embedding models that provide their weights in the [safetensors](https://github.com/huggingface/safetensors) format** (i.e., the model repo must contain a `model.safetensors` file). If the model repo does not provide the required file, loading will fail with a clear error. Most official BERT and DistilBERT models do **not** provide safetensors; many Sentence Transformers and JinaBERT models do.
+Red-Candle **only supports embedding models and rerankers that provide their weights in the [safetensors](https://github.com/huggingface/safetensors) format** (i.e., the model repo must contain a `model.safetensors` file). If the model repo does not provide the required file, loading will fail with a clear error. Most official BERT and DistilBERT models do **not** provide safetensors; many Sentence Transformers and JinaBERT models do.
 
 **If you encounter an error like:**
 
@@ -178,13 +182,22 @@ RuntimeError: model.safetensors not found after download. Only safetensors model
 
 this means the selected model is not compatible. Please choose a model repo that provides the required file.
 
+### LLMs: Safetensors and GGUF Support
+
+LLM models support two formats:
+1. **Safetensors format** - Standard HuggingFace models (e.g., `TinyLlama/TinyLlama-1.1B-Chat-v1.0`)
+2. **GGUF quantized format** - Memory-efficient quantized models (e.g., `TheBloke/Llama-2-7B-Chat-GGUF`)
+
+See the [Quantized Model Support](#quantized-model-support-gguf) section for details on using GGUF models.
+
 ## Supported Embedding Models
 
 Red-Candle supports the following embedding model types from Hugging Face:
 
 1. `Candle::EmbeddingModelType::JINA_BERT` - Jina BERT models (e.g., `jinaai/jina-embeddings-v2-base-en`) (**safetensors required**)
-2. `Candle::EmbeddingModelType::STANDARD_BERT` - Standard BERT models (e.g., `sentence-transformers/all-MiniLM-L6-v2`) (**safetensors required**)
+2. `Candle::EmbeddingModelType::MINILM` - MINILM models (e.g., `sentence-transformers/all-MiniLM-L6-v2`) (**safetensors required**)
 3. `Candle::EmbeddingModelType::DISTILBERT` - DistilBERT models (e.g., `distilbert-base-uncased-finetuned-sst-2-english`) (**safetensors required**)
+4. `Candle::EmbeddingModelType::STANDARD_BERT` - Standard BERT models (e.g., `scientistcom/BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext`) (**safetensors required**)
 
 > **Note:** Most official BERT and DistilBERT models do _not_ provide safetensors. Please check the model repo before use.
 
@@ -260,7 +273,7 @@ ranked_results = reranker.rerank(query, documents, pooling_method: "pooler", app
 # Or apply sigmoid activation to get scores between 0 and 1
 sigmoid_results = reranker.rerank(query, documents, pooling_method: "pooler", apply_sigmoid: true)
 
-# The pooler method is the default and is recommended for cross-encoders, as is apply_sigmod, so the above is the same as:
+# The pooler method is the default and is recommended for cross-encoders, as is apply_sigmoid, so the above is the same as:
 ranked_results = reranker.rerank(query, documents)
 
 # Results are returned as an array of hashes, sorted by relevance
@@ -351,6 +364,311 @@ The reranker uses a BERT-based architecture that:
 
 This joint processing allows cross-encoders to capture subtle semantic relationships between queries and documents, making them more accurate for reranking tasks, though at the cost of higher computational requirements.
 
+## Tokenizer
+
+Red-Candle provides direct access to tokenizers for text preprocessing and analysis. This is useful for understanding how models process text, debugging issues, and building custom NLP pipelines.
+
+### Basic Usage
+
+```ruby
+require 'candle'
+
+# Load a tokenizer from HuggingFace
+tokenizer = Candle::Tokenizer.from_pretrained("bert-base-uncased")
+
+# Encode text to token IDs
+token_ids = tokenizer.encode("Hello, world!")
+# => [101, 7592, 1010, 2088, 999, 102]
+
+# Decode token IDs back to text
+text = tokenizer.decode(token_ids)
+# => "hello, world!"
+
+# Get token strings (subwords) - useful for visualization
+tokens = tokenizer.encode_to_tokens("Hello, world!")
+# => ["[CLS]", "hello", ",", "world", "!", "[SEP]"]
+
+# Get both IDs and tokens together
+result = tokenizer.encode_with_tokens("preprocessing")
+# => {"ids" => [101, 3653, 22618, 2527, 102], 
+#     "tokens" => ["[CLS]", "prep", "##ro", "##ces", "##sing", "[SEP]"]}
+```
+
+### Batch Processing
+
+```ruby
+# Encode multiple texts at once
+texts = ["Hello world", "How are you?", "Tokenizers are cool"]
+batch_ids = tokenizer.encode_batch(texts)
+
+# Get token strings for multiple texts
+batch_tokens = tokenizer.encode_batch_to_tokens(texts)
+```
+
+### Vocabulary Access
+
+```ruby
+# Get vocabulary size
+vocab_size = tokenizer.vocab_size
+# => 30522
+
+# Get full vocabulary as a hash
+vocab = tokenizer.get_vocab
+# vocab["hello"] => 7592
+
+# Convert a specific token ID to its string
+token_str = tokenizer.id_to_token(7592)
+# => "hello"
+
+# Get special tokens
+special = tokenizer.get_special_tokens
+# => {"cls_token" => 101, "sep_token" => 102, "pad_token" => 0, ...}
+```
+
+### Configuration
+
+```ruby
+# Create a tokenizer with padding enabled
+padded_tokenizer = tokenizer.with_padding(length: 128)
+
+# Create a tokenizer with truncation
+truncated_tokenizer = tokenizer.with_truncation(512)
+
+# Configure padding with more options
+padded_tokenizer = tokenizer.with_padding(
+  length: 128,          # Fixed length padding
+  direction: "right",   # Pad on the right (default)
+  pad_token: "[PAD]"    # Padding token
+)
+```
+
+### Model Integration
+
+All models expose their tokenizers:
+
+```ruby
+# From LLM
+llm = Candle::LLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
+llm_tokenizer = llm.tokenizer
+
+# From EmbeddingModel
+embedding_model = Candle::EmbeddingModel.new
+emb_tokenizer = embedding_model.tokenizer
+
+# From Reranker
+reranker = Candle::Reranker.new(model_path: "cross-encoder/ms-marco-MiniLM-L-12-v2")
+rank_tokenizer = reranker.tokenizer
+```
+
+### Understanding Subword Tokenization
+
+Modern tokenizers split unknown or rare words into subword pieces:
+
+```ruby
+# See how words are split into subwords
+result = tokenizer.encode_with_tokens("unbelievable")
+# => {"ids" => [101, 4895, 6499, 102], 
+#     "tokens" => ["[CLS]", "un", "##believable", "[SEP]"]}
+
+# The ## prefix indicates a continuation of the previous token
+complex = tokenizer.encode_to_tokens("preprocessing tokenization")
+# => ["[CLS]", "prep", "##ro", "##ces", "##sing", "token", "##ization", "[SEP]"]
+```
+
+### Use Cases
+
+- **Token Analysis**: Understand how your text is being processed by models
+- **Debugging**: See why certain inputs might cause unexpected model behavior  
+- **Custom Preprocessing**: Build your own text processing pipelines
+- **Educational**: Teach how modern NLP models handle text
+- **NER Preparation**: Get aligned tokens for named entity recognition tasks
+
+## Named Entity Recognition (NER)
+
+Red-Candle includes comprehensive Named Entity Recognition capabilities for extracting entities like people, organizations, locations, and custom entity types from text.
+
+### Model-based NER
+
+Load pre-trained NER models from HuggingFace:
+
+```ruby
+require 'candle'
+
+# Load a pre-trained NER model
+ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner")
+
+# Or load a model with a specific tokenizer (for models without tokenizer.json)
+ner = Candle::NER.from_pretrained("dslim/bert-base-NER", tokenizer: "bert-base-cased")
+
+# Extract entities from text
+text = "Apple Inc. was founded by Steve Jobs and Steve Wozniak in Cupertino, California."
+entities = ner.extract_entities(text)
+
+entities.each do |entity|
+  puts "#{entity['text']} (#{entity['label']}) - confidence: #{entity['confidence'].round(2)}"
+end
+# Output:
+# Apple Inc. (ORG) - confidence: 0.99
+# Steve Jobs (PER) - confidence: 0.99
+# Steve Wozniak (PER) - confidence: 0.98
+# Cupertino (LOC) - confidence: 0.97
+# California (LOC) - confidence: 0.98
+
+# Adjust confidence threshold (default: 0.9)
+entities = ner.extract_entities(text, confidence_threshold: 0.95)
+
+# Get token-level predictions for detailed analysis
+tokens = ner.predict_tokens(text)
+```
+
+### Pattern-based Recognition
+
+For domain-specific entities, use regex patterns:
+
+```ruby
+# Create pattern-based recognizers
+email_recognizer = Candle::PatternEntityRecognizer.new("EMAIL", [
+  /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/
+])
+
+phone_recognizer = Candle::PatternEntityRecognizer.new("PHONE", [
+  /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/,         # 555-123-4567
+  /\b\(\d{3}\)\s*\d{3}[-.]?\d{4}\b/,      # (555) 123-4567
+  /\b\+1\s*\d{3}[-.]?\d{3}[-.]?\d{4}\b/   # +1 555-123-4567
+])
+
+# Extract entities
+text = "Contact us at info@example.com or call 555-123-4567"
+email_entities = email_recognizer.recognize(text)
+phone_entities = phone_recognizer.recognize(text)
+```
+
+### Gazetteer-based Recognition
+
+Use dictionaries for known entities:
+
+```ruby
+# Create gazetteer recognizers
+companies = ["Apple", "Google", "Microsoft", "Amazon", "Tesla"]
+company_recognizer = Candle::GazetteerEntityRecognizer.new("COMPANY", companies)
+
+# Load from file
+drug_recognizer = Candle::GazetteerEntityRecognizer.new("DRUG")
+drug_recognizer.load_from_file("drug_names.txt")
+
+# Case-sensitive matching
+product_recognizer = Candle::GazetteerEntityRecognizer.new("PRODUCT", 
+  ["iPhone", "iPad", "MacBook"], 
+  case_sensitive: true
+)
+```
+
+### Hybrid NER
+
+Combine ML models with rule-based approaches for best results:
+
+```ruby
+# Create hybrid NER system
+hybrid = Candle::HybridNER.new("Babelscape/wikineural-multilingual-ner")
+
+# Add pattern recognizers
+hybrid.add_pattern_recognizer("EMAIL", [/\b[\w._%+-]+@[\w.-]+\.[A-Z|a-z]{2,}\b/])
+hybrid.add_pattern_recognizer("PHONE", [/\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/])
+
+# Add gazetteer recognizers  
+hybrid.add_gazetteer_recognizer("COMPANY", ["Apple", "Google", "Microsoft"])
+hybrid.add_gazetteer_recognizer("PRODUCT", ["iPhone", "Android", "Windows"])
+
+# Extract all entities
+text = "John Smith (john@apple.com) from Apple called about the new iPhone. Reach him at 555-0123."
+entities = hybrid.extract_entities(text)
+
+# Results include entities from all recognizers
+# Overlapping entities are automatically resolved (highest confidence wins)
+```
+
+### Custom Entity Types
+
+Perfect for specialized domains:
+
+```ruby
+# Biomedical entities
+gene_patterns = [
+  /\b[A-Z][A-Z0-9]{2,}\b/,      # TP53, BRCA1, EGFR
+  /\bCD\d+\b/,                  # CD4, CD8, CD34
+  /\b[A-Z]+\d[A-Z]\d*\b/        # RAD51C, PALB2
+]
+gene_recognizer = Candle::PatternEntityRecognizer.new("GENE", gene_patterns)
+
+# Financial entities
+ticker_patterns = [
+  /\$[A-Z]{1,5}\b/,             # $AAPL, $GOOGL
+  /\b[A-Z]{1,5}\.NYSE\b/,       # AAPL.NYSE
+  /\b[A-Z]{1,5}\.NASDAQ\b/      # GOOGL.NASDAQ
+]
+ticker_recognizer = Candle::PatternEntityRecognizer.new("TICKER", ticker_patterns)
+
+# Legal entities
+case_patterns = [
+  /\b\d+\s+F\.\d+\s+\d+\b/,     # 123 F.3d 456
+  /\b\d+\s+U\.S\.\s+\d+\b/,     # 123 U.S. 456
+  /\bNo\.\s+\d+-\d+\b/          # No. 20-1234
+]
+case_recognizer = Candle::PatternEntityRecognizer.new("CASE", case_patterns)
+```
+
+### Available Pre-trained Models
+
+Popular NER models on HuggingFace:
+
+```ruby
+# General multilingual NER (4 entity types: PER, ORG, LOC, MISC)
+ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner")
+
+# English NER (requires separate tokenizer)
+ner = Candle::NER.from_pretrained("dslim/bert-base-NER", tokenizer: "bert-base-cased")
+
+# Multilingual NER  
+ner = Candle::NER.from_pretrained("Davlan/bert-base-multilingual-cased-ner-hrl")
+
+# OntoNotes 5 (18 entity types including DATE, TIME, MONEY, etc.)
+ner = Candle::NER.from_pretrained("flair/ner-english-ontonotes-large")
+
+# Biomedical NER
+ner = Candle::NER.from_pretrained("dmis-lab/biobert-base-cased-v1.2")
+ner = Candle::NER.from_pretrained("allenai/scibert_scivocab_uncased")
+```
+
+### Performance Tips
+
+1. **Device Selection**: Use GPU for faster inference
+   ```ruby
+   ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner", device: Candle::Device.metal)
+   ```
+
+2. **Batch Processing**: Process multiple texts together when possible
+   
+3. **Confidence Threshold**: Balance precision/recall with appropriate thresholds
+   
+4. **Entity Resolution**: The hybrid NER automatically handles overlapping entities
+
+### Output Format
+
+All NER methods return entities in a consistent format:
+
+```ruby
+{
+  "text" => "Apple Inc.",          # The entity text
+  "label" => "ORG",               # Entity type
+  "start" => 0,                   # Character start position
+  "end" => 10,                    # Character end position  
+  "confidence" => 0.99,           # Confidence score (0-1)
+  "token_start" => 0,             # Token start index (model-based only)
+  "token_end" => 2,               # Token end index (model-based only)
+  "source" => "model"             # Source: "model", "pattern", or "gazetteer"
+}
+```
+
 ## Common Runtime Errors
 
 ### 1. Weight is negative, too large or not a valid number

data/ext/candle/src/lib.rs

@@ -1,11 +1,13 @@
 use magnus::{function, prelude::*, Ruby};
 
 use crate::ruby::candle_utils;
-use crate::ruby::Result as RbResult;
+use crate::ruby::Result;
 
 pub mod llm;
+pub mod ner;
 pub mod reranker;
 pub mod ruby;
+pub mod tokenizer;
 
 // Configuration detection from build.rs
 #[cfg(all(has_metal, not(force_cpu)))]
@@ -33,7 +35,7 @@ pub fn get_build_info() -> magnus::RHash {
 }
 
 #[magnus::init]
-fn init(ruby: &Ruby) -> RbResult<()> {
+fn init(ruby: &Ruby) -> Result<()> {
     let rb_candle = ruby.define_module("Candle")?;
     
     // Export build info
@@ -41,11 +43,12 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     
     ruby::init_embedding_model(rb_candle)?;
     ruby::init_llm(rb_candle)?;
+    ner::init(rb_candle)?;
     reranker::init(rb_candle)?;
     ruby::dtype::init(rb_candle)?;
-    ruby::qtensor::init(rb_candle)?;
     ruby::device::init(rb_candle)?;
     ruby::tensor::init(rb_candle)?;
+    ruby::tokenizer::init(rb_candle)?;
     candle_utils(rb_candle)?;
 
     Ok(())

data/ext/candle/src/llm/gemma.rs

@@ -21,6 +21,11 @@ impl Gemma {
         self.model.clear_kv_cache();
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Gemma model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()

data/ext/candle/src/llm/llama.rs

@@ -28,6 +28,11 @@ impl Llama {
         }
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Llama model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()

data/ext/candle/src/llm/mistral.rs

@@ -21,6 +21,11 @@ impl Mistral {
         self.model.clear_kv_cache();
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Mistral model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()

data/ext/candle/src/llm/mod.rs

@@ -1,5 +1,4 @@
 use candle_core::{Device, Result as CandleResult};
-use tokenizers::Tokenizer;
 
 pub mod mistral;
 pub mod llama;
@@ -11,6 +10,7 @@ pub mod quantized_gguf;
 pub use generation_config::GenerationConfig;
 pub use text_generation::TextGeneration;
 pub use quantized_gguf::QuantizedGGUF;
+pub use crate::tokenizer::TokenizerWrapper;
 
 /// Trait for text generation models
 pub trait TextGenerator: Send + Sync {
@@ -37,92 +37,4 @@ pub trait TextGenerator: Send + Sync {
     
     /// Clear any cached state (like KV cache)
     fn clear_cache(&mut self);
-}
-
-/// Common structure for managing tokenizer
-#[derive(Debug)]
-pub struct TokenizerWrapper {
-    tokenizer: Tokenizer,
-}
-
-impl TokenizerWrapper {
-    pub fn new(tokenizer: Tokenizer) -> Self {
-        Self { tokenizer }
-    }
-
-    pub fn encode(&self, text: &str, add_special_tokens: bool) -> CandleResult<Vec<u32>> {
-        let encoding = self.tokenizer
-            .encode(text, add_special_tokens)
-            .map_err(|e| candle_core::Error::Msg(format!("Tokenizer error: {}", e)))?;
-        Ok(encoding.get_ids().to_vec())
-    }
-
-    pub fn decode(&self, tokens: &[u32], skip_special_tokens: bool) -> CandleResult<String> {
-        self.tokenizer
-            .decode(tokens, skip_special_tokens)
-            .map_err(|e| candle_core::Error::Msg(format!("Tokenizer decode error: {}", e)))
-    }
-
-    pub fn token_to_piece(&self, token: u32) -> CandleResult<String> {
-        self.tokenizer
-            .id_to_token(token)
-            .map(|s| s.to_string())
-            .ok_or_else(|| candle_core::Error::Msg(format!("Unknown token id: {}", token)))
-    }
-    
-    /// Decode a single token for streaming output
-    pub fn decode_token(&self, token: u32) -> CandleResult<String> {
-        // Decode the single token properly
-        self.decode(&[token], true)
-    }
-    
-    /// Decode tokens incrementally for streaming
-    /// This is more efficient than decoding single tokens
-    pub fn decode_incremental(&self, all_tokens: &[u32], new_tokens_start: usize) -> CandleResult<String> {
-        if new_tokens_start >= all_tokens.len() {
-            return Ok(String::new());
-        }
-        
-        // Decode all tokens up to this point
-        let full_text = self.decode(all_tokens, true)?;
-        
-        // If we're at the start, return everything
-        if new_tokens_start == 0 {
-            return Ok(full_text);
-        }
-        
-        // Otherwise, decode up to the previous token and return the difference
-        let previous_text = self.decode(&all_tokens[..new_tokens_start], true)?;
-        
-        // Find the common prefix between the two strings to handle cases where
-        // the tokenizer might produce slightly different text when decoding
-        // different token sequences
-        let common_prefix_len = full_text
-            .char_indices()
-            .zip(previous_text.chars())
-            .take_while(|((_, c1), c2)| c1 == c2)
-            .count();
-        
-        // Find the byte position of the character boundary
-        let byte_pos = full_text
-            .char_indices()
-            .nth(common_prefix_len)
-            .map(|(pos, _)| pos)
-            .unwrap_or(full_text.len());
-        
-        // Return only the new portion
-        Ok(full_text[byte_pos..].to_string())
-    }
-    
-    /// Format tokens with debug information
-    pub fn format_tokens_with_debug(&self, tokens: &[u32]) -> CandleResult<String> {
-        let mut result = String::new();
-        
-        for &token in tokens {
-            let token_piece = self.token_to_piece(token)?;
-            result.push_str(&format!("[{}:{}]", token, token_piece));
-        }
-        
-        Ok(result)
-    }
 }

data/ext/candle/src/llm/quantized_gguf.rs

@@ -28,6 +28,11 @@ enum ModelType {
 }
 
 impl QuantizedGGUF {
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a quantized model from a GGUF file
     pub async fn from_pretrained(model_id: &str, device: Device, tokenizer_source: Option<&str>) -> CandleResult<Self> {
         // Check if user specified an exact GGUF filename