red-candle 1.0.0.pre.7 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bea0c9f27d5bbbc43e0c2e8fa5456b2b79511f164f7083512025538ca172077
-  data.tar.gz: b591f559c63c1bdb9fa96beaf1079186284b219e2ceba50d8e2891fa4f3096bd
+  metadata.gz: 52c635f005d25a305f99781763a4a3cc03f85fc5b74f0e576e51973ef8306fac
+  data.tar.gz: 1a0ac260a3803f1920ba2d9f71ec361013ae1eb99cf2caed62c0e9aecc583e96
 SHA512:
-  metadata.gz: 380cde32abf995f9766056638d4d0612acef746a84b31451c0aa2fb8cd205fc76b76b8909729a9af267ee6c549f9149253b2bfb70717385b5afe2cd2ad0b3152
-  data.tar.gz: 7acdfc4de263501be51e3f74933c528b9d723b4da93e78e3abae2454dc2746fb7696c09722fe6666084a48dff098501b2d1af503963b5d739e879bd0fcb96e43
+  metadata.gz: d301e6ed0fe8ac144c0735288c687f5dd74e7967dbe5d357e550ca5d467f6a33017b2bd9e7f46081711b6bf13555caa3e044183cd74cfaa89151e15c8cdb04a4
+  data.tar.gz: d296c35002b6d0ed919176375e5cc5d93c70fae0c0ae9a02d5cf86b8a4a49a67898c7fbc96e16350bba6792b18792126c976de156fb854b9b4f3260fa052cd79

data/Gemfile

@@ -1,12 +1,3 @@
 source "https://rubygems.org"
 
-gemspec
-
-gem "minitest"
-gem "rake"
-gem "rake-compiler"
-
-gem "yard", require: false
-gem "yard-rustdoc", require: false
-
-gem "redcarpet", "~> 3.6"
+gemspec

data/README.md

@@ -1,9 +1,66 @@
-# red-candle
+# `red-candle` Native LLMs for Ruby 🚀
 
 [![build](https://github.com/assaydepot/red-candle/actions/workflows/build.yml/badge.svg)](https://github.com/assaydepot/red-candle/actions/workflows/build.yml)
 [![Gem Version](https://badge.fury.io/rb/red-candle.svg)](https://badge.fury.io/rb/red-candle)
 
- [candle](https://github.com/huggingface/candle) - Minimalist ML framework - for Ruby
+Run state-of-the-art **language models directly from Ruby**. No Python, no APIs, no external services - just Ruby with blazing-fast Rust under the hood. Hardware accelerated with **Metal (Mac)** and **CUDA (NVIDIA).**
+
+## Install & Chat in 30 Seconds
+
+[![red-candle quickstart](https://img.youtube.com/vi/hbyFCyh8esk/0.jpg)](https://www.youtube.com/watch?v=hbyFCyh8esk)
+
+```bash
+# Install the gem
+gem install red-candle
+```
+
+```ruby
+require 'candle'
+
+# Download a model (one-time, ~650MB) - Mistral, Llama3, Gemma all work!
+llm = Candle::LLM.from_pretrained("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", 
+                                  gguf_file: "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf")
+
+# Chat with it - no API calls, running locally in your Ruby process!
+messages = [
+  { role: "user", content: "Explain Ruby in one sentence" }
+]
+
+puts llm.chat(messages)
+# => "Ruby is a dynamic, object-oriented programming language known for its 
+#     simplicity, elegance, and productivity, often used for web development 
+#     with frameworks like Rails."
+```
+
+## What Just Happened?
+
+You just ran a 1.1-billion parameter AI model inside Ruby. The model lives in your process memory, runs on your hardware (CPU/GPU), and responds instantly without network latency.
+
+## Stream Responses Like a Pro
+
+```ruby
+# Watch the AI think in real-time
+llm.chat_stream(messages) do |token|
+  print token
+end
+```
+
+## Why This Matters
+
+- **Privacy**: Your data never leaves your machine
+- **Speed**: No network overhead, direct memory access
+- **Control**: Fine-tune generation parameters, access raw tokens
+- **Integration**: It's just Ruby objects - use it anywhere Ruby runs
+
+## Supports
+
+- **Tokenizers**: Access the tokenizer directly
+- **EmbeddingModel**: Generate embeddings for text
+- **Reranker**: Rerank documents based on relevance
+- **NER**: Named Entity Recognition directly from Ruby
+- **LLM**: Chat with Large Language Models (e.g., Llama, Mistral, Gemma)
+
+----
 
 ## Usage
 
@@ -127,6 +184,8 @@ response = llm.chat(messages)
 
 ### GPU Acceleration
 
+We see an 18x speed up running LLMs under CUDA vs CPU and a >3x speed up running under Metal vs CPU. Details [here](DEVICE_SUPPORT.md#performance-considerations).
+
 ```ruby
 # CPU works for all models
 device = Candle::Device.cpu
@@ -166,9 +225,11 @@ This is particularly useful for:
 - Troubleshooting generation problems
 - Analyzing model behavior
 
-## ⚠️ Model Format Requirement: Safetensors Only
+## ⚠️ Model Format Requirements
 
-Red-Candle **only supports embedding models that provide their weights in the [safetensors](https://github.com/huggingface/safetensors) format** (i.e., the model repo must contain a `model.safetensors` file). If the model repo does not provide the required file, loading will fail with a clear error. Most official BERT and DistilBERT models do **not** provide safetensors; many Sentence Transformers and JinaBERT models do.
+### EmbeddingModels and Rerankers: Safetensors Only
+
+Red-Candle **only supports embedding models and rerankers that provide their weights in the [safetensors](https://github.com/huggingface/safetensors) format** (i.e., the model repo must contain a `model.safetensors` file). If the model repo does not provide the required file, loading will fail with a clear error. Most official BERT and DistilBERT models do **not** provide safetensors; many Sentence Transformers and JinaBERT models do.
 
 **If you encounter an error like:**
 
@@ -178,13 +239,22 @@ RuntimeError: model.safetensors not found after download. Only safetensors model
 
 this means the selected model is not compatible. Please choose a model repo that provides the required file.
 
+### LLMs: Safetensors and GGUF Support
+
+LLM models support two formats:
+1. **Safetensors format** - Standard HuggingFace models (e.g., `TinyLlama/TinyLlama-1.1B-Chat-v1.0`)
+2. **GGUF quantized format** - Memory-efficient quantized models (e.g., `TheBloke/Llama-2-7B-Chat-GGUF`)
+
+See the [Quantized Model Support](#quantized-model-support-gguf) section for details on using GGUF models.
+
 ## Supported Embedding Models
 
 Red-Candle supports the following embedding model types from Hugging Face:
 
 1. `Candle::EmbeddingModelType::JINA_BERT` - Jina BERT models (e.g., `jinaai/jina-embeddings-v2-base-en`) (**safetensors required**)
-2. `Candle::EmbeddingModelType::STANDARD_BERT` - Standard BERT models (e.g., `sentence-transformers/all-MiniLM-L6-v2`) (**safetensors required**)
+2. `Candle::EmbeddingModelType::MINILM` - MINILM models (e.g., `sentence-transformers/all-MiniLM-L6-v2`) (**safetensors required**)
 3. `Candle::EmbeddingModelType::DISTILBERT` - DistilBERT models (e.g., `distilbert-base-uncased-finetuned-sst-2-english`) (**safetensors required**)
+4. `Candle::EmbeddingModelType::STANDARD_BERT` - Standard BERT models (e.g., `scientistcom/BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext`) (**safetensors required**)
 
 > **Note:** Most official BERT and DistilBERT models do _not_ provide safetensors. Please check the model repo before use.
 
@@ -260,7 +330,7 @@ ranked_results = reranker.rerank(query, documents, pooling_method: "pooler", app
 # Or apply sigmoid activation to get scores between 0 and 1
 sigmoid_results = reranker.rerank(query, documents, pooling_method: "pooler", apply_sigmoid: true)
 
-# The pooler method is the default and is recommended for cross-encoders, as is apply_sigmod, so the above is the same as:
+# The pooler method is the default and is recommended for cross-encoders, as is apply_sigmoid, so the above is the same as:
 ranked_results = reranker.rerank(query, documents)
 
 # Results are returned as an array of hashes, sorted by relevance
@@ -351,9 +421,314 @@ The reranker uses a BERT-based architecture that:
 
 This joint processing allows cross-encoders to capture subtle semantic relationships between queries and documents, making them more accurate for reranking tasks, though at the cost of higher computational requirements.
 
+## Tokenizer
+
+Red-Candle provides direct access to tokenizers for text preprocessing and analysis. This is useful for understanding how models process text, debugging issues, and building custom NLP pipelines.
+
+### Basic Usage
+
+```ruby
+require 'candle'
+
+# Load a tokenizer from HuggingFace
+tokenizer = Candle::Tokenizer.from_pretrained("bert-base-uncased")
+
+# Encode text to token IDs
+token_ids = tokenizer.encode("Hello, world!")
+# => [101, 7592, 1010, 2088, 999, 102]
+
+# Decode token IDs back to text
+text = tokenizer.decode(token_ids)
+# => "hello, world!"
+
+# Get token strings (subwords) - useful for visualization
+tokens = tokenizer.encode_to_tokens("Hello, world!")
+# => ["[CLS]", "hello", ",", "world", "!", "[SEP]"]
+
+# Get both IDs and tokens together
+result = tokenizer.encode_with_tokens("preprocessing")
+# => {"ids" => [101, 3653, 22618, 2527, 102], 
+#     "tokens" => ["[CLS]", "prep", "##ro", "##ces", "##sing", "[SEP]"]}
+```
+
+### Batch Processing
+
+```ruby
+# Encode multiple texts at once
+texts = ["Hello world", "How are you?", "Tokenizers are cool"]
+batch_ids = tokenizer.encode_batch(texts)
+
+# Get token strings for multiple texts
+batch_tokens = tokenizer.encode_batch_to_tokens(texts)
+```
+
+### Vocabulary Access
+
+```ruby
+# Get vocabulary size
+vocab_size = tokenizer.vocab_size
+# => 30522
+
+# Get full vocabulary as a hash
+vocab = tokenizer.get_vocab
+# vocab["hello"] => 7592
+
+# Convert a specific token ID to its string
+token_str = tokenizer.id_to_token(7592)
+# => "hello"
+
+# Get special tokens
+special = tokenizer.get_special_tokens
+# => {"cls_token" => 101, "sep_token" => 102, "pad_token" => 0, ...}
+```
+
+### Configuration
+
+```ruby
+# Create a tokenizer with padding enabled
+padded_tokenizer = tokenizer.with_padding(length: 128)
+
+# Create a tokenizer with truncation
+truncated_tokenizer = tokenizer.with_truncation(512)
+
+# Configure padding with more options
+padded_tokenizer = tokenizer.with_padding(
+  length: 128,          # Fixed length padding
+  direction: "right",   # Pad on the right (default)
+  pad_token: "[PAD]"    # Padding token
+)
+```
+
+### Model Integration
+
+All models expose their tokenizers:
+
+```ruby
+# From LLM
+llm = Candle::LLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
+llm_tokenizer = llm.tokenizer
+
+# From EmbeddingModel
+embedding_model = Candle::EmbeddingModel.new
+emb_tokenizer = embedding_model.tokenizer
+
+# From Reranker
+reranker = Candle::Reranker.new(model_path: "cross-encoder/ms-marco-MiniLM-L-12-v2")
+rank_tokenizer = reranker.tokenizer
+```
+
+### Understanding Subword Tokenization
+
+Modern tokenizers split unknown or rare words into subword pieces:
+
+```ruby
+# See how words are split into subwords
+result = tokenizer.encode_with_tokens("unbelievable")
+# => {"ids" => [101, 4895, 6499, 102], 
+#     "tokens" => ["[CLS]", "un", "##believable", "[SEP]"]}
+
+# The ## prefix indicates a continuation of the previous token
+complex = tokenizer.encode_to_tokens("preprocessing tokenization")
+# => ["[CLS]", "prep", "##ro", "##ces", "##sing", "token", "##ization", "[SEP]"]
+```
+
+### Use Cases
+
+- **Token Analysis**: Understand how your text is being processed by models
+- **Debugging**: See why certain inputs might cause unexpected model behavior  
+- **Custom Preprocessing**: Build your own text processing pipelines
+- **Educational**: Teach how modern NLP models handle text
+- **NER Preparation**: Get aligned tokens for named entity recognition tasks
+
+## Named Entity Recognition (NER)
+
+Red-Candle includes comprehensive Named Entity Recognition capabilities for extracting entities like people, organizations, locations, and custom entity types from text.
+
+### Model-based NER
+
+Load pre-trained NER models from HuggingFace:
+
+```ruby
+require 'candle'
+
+# Load a pre-trained NER model
+ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner")
+
+# Or load a model with a specific tokenizer (for models without tokenizer.json)
+ner = Candle::NER.from_pretrained("dslim/bert-base-NER", tokenizer: "bert-base-cased")
+
+# Extract entities from text
+text = "Apple Inc. was founded by Steve Jobs and Steve Wozniak in Cupertino, California."
+entities = ner.extract_entities(text)
+
+entities.each do |entity|
+  puts "#{entity['text']} (#{entity['label']}) - confidence: #{entity['confidence'].round(2)}"
+end
+# Output:
+# Apple Inc. (ORG) - confidence: 0.99
+# Steve Jobs (PER) - confidence: 0.99
+# Steve Wozniak (PER) - confidence: 0.98
+# Cupertino (LOC) - confidence: 0.97
+# California (LOC) - confidence: 0.98
+
+# Adjust confidence threshold (default: 0.9)
+entities = ner.extract_entities(text, confidence_threshold: 0.95)
+
+# Get token-level predictions for detailed analysis
+tokens = ner.predict_tokens(text)
+```
+
+### Pattern-based Recognition
+
+For domain-specific entities, use regex patterns:
+
+```ruby
+# Create pattern-based recognizers
+email_recognizer = Candle::PatternEntityRecognizer.new("EMAIL", [
+  /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/
+])
+
+phone_recognizer = Candle::PatternEntityRecognizer.new("PHONE", [
+  /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/,         # 555-123-4567
+  /\b\(\d{3}\)\s*\d{3}[-.]?\d{4}\b/,      # (555) 123-4567
+  /\b\+1\s*\d{3}[-.]?\d{3}[-.]?\d{4}\b/   # +1 555-123-4567
+])
+
+# Extract entities
+text = "Contact us at info@example.com or call 555-123-4567"
+email_entities = email_recognizer.recognize(text)
+phone_entities = phone_recognizer.recognize(text)
+```
+
+### Gazetteer-based Recognition
+
+Use dictionaries for known entities:
+
+```ruby
+# Create gazetteer recognizers
+companies = ["Apple", "Google", "Microsoft", "Amazon", "Tesla"]
+company_recognizer = Candle::GazetteerEntityRecognizer.new("COMPANY", companies)
+
+# Load from file
+drug_recognizer = Candle::GazetteerEntityRecognizer.new("DRUG")
+drug_recognizer.load_from_file("drug_names.txt")
+
+# Case-sensitive matching
+product_recognizer = Candle::GazetteerEntityRecognizer.new("PRODUCT", 
+  ["iPhone", "iPad", "MacBook"], 
+  case_sensitive: true
+)
+```
+
+### Hybrid NER
+
+Combine ML models with rule-based approaches for best results:
+
+```ruby
+# Create hybrid NER system
+hybrid = Candle::HybridNER.new("Babelscape/wikineural-multilingual-ner")
+
+# Add pattern recognizers
+hybrid.add_pattern_recognizer("EMAIL", [/\b[\w._%+-]+@[\w.-]+\.[A-Z|a-z]{2,}\b/])
+hybrid.add_pattern_recognizer("PHONE", [/\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/])
+
+# Add gazetteer recognizers  
+hybrid.add_gazetteer_recognizer("COMPANY", ["Apple", "Google", "Microsoft"])
+hybrid.add_gazetteer_recognizer("PRODUCT", ["iPhone", "Android", "Windows"])
+
+# Extract all entities
+text = "John Smith (john@apple.com) from Apple called about the new iPhone. Reach him at 555-0123."
+entities = hybrid.extract_entities(text)
+
+# Results include entities from all recognizers
+# Overlapping entities are automatically resolved (highest confidence wins)
+```
+
+### Custom Entity Types
+
+Perfect for specialized domains:
+
+```ruby
+# Biomedical entities
+gene_patterns = [
+  /\b[A-Z][A-Z0-9]{2,}\b/,      # TP53, BRCA1, EGFR
+  /\bCD\d+\b/,                  # CD4, CD8, CD34
+  /\b[A-Z]+\d[A-Z]\d*\b/        # RAD51C, PALB2
+]
+gene_recognizer = Candle::PatternEntityRecognizer.new("GENE", gene_patterns)
+
+# Financial entities
+ticker_patterns = [
+  /\$[A-Z]{1,5}\b/,             # $AAPL, $GOOGL
+  /\b[A-Z]{1,5}\.NYSE\b/,       # AAPL.NYSE
+  /\b[A-Z]{1,5}\.NASDAQ\b/      # GOOGL.NASDAQ
+]
+ticker_recognizer = Candle::PatternEntityRecognizer.new("TICKER", ticker_patterns)
+
+# Legal entities
+case_patterns = [
+  /\b\d+\s+F\.\d+\s+\d+\b/,     # 123 F.3d 456
+  /\b\d+\s+U\.S\.\s+\d+\b/,     # 123 U.S. 456
+  /\bNo\.\s+\d+-\d+\b/          # No. 20-1234
+]
+case_recognizer = Candle::PatternEntityRecognizer.new("CASE", case_patterns)
+```
+
+### Available Pre-trained Models
+
+Popular NER models on HuggingFace:
+
+```ruby
+# General multilingual NER (4 entity types: PER, ORG, LOC, MISC)
+ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner")
+
+# English NER (requires separate tokenizer)
+ner = Candle::NER.from_pretrained("dslim/bert-base-NER", tokenizer: "bert-base-cased")
+
+# Multilingual NER  
+ner = Candle::NER.from_pretrained("Davlan/bert-base-multilingual-cased-ner-hrl")
+
+# OntoNotes 5 (18 entity types including DATE, TIME, MONEY, etc.)
+ner = Candle::NER.from_pretrained("flair/ner-english-ontonotes-large")
+
+# Biomedical NER
+ner = Candle::NER.from_pretrained("dmis-lab/biobert-base-cased-v1.2")
+ner = Candle::NER.from_pretrained("allenai/scibert_scivocab_uncased")
+```
+
+### Performance Tips
+
+1. **Device Selection**: Use GPU for faster inference
+   ```ruby
+   ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner", device: Candle::Device.metal)
+   ```
+
+2. **Batch Processing**: Process multiple texts together when possible
+   
+3. **Confidence Threshold**: Balance precision/recall with appropriate thresholds
+   
+4. **Entity Resolution**: The hybrid NER automatically handles overlapping entities
+
+### Output Format
+
+All NER methods return entities in a consistent format:
+
+```ruby
+{
+  "text" => "Apple Inc.",          # The entity text
+  "label" => "ORG",               # Entity type
+  "start" => 0,                   # Character start position
+  "end" => 10,                    # Character end position  
+  "confidence" => 0.99,           # Confidence score (0-1)
+  "token_start" => 0,             # Token start index (model-based only)
+  "token_end" => 2,               # Token end index (model-based only)
+  "source" => "model"             # Source: "model", "pattern", or "gazetteer"
+}
+```
+
 ## Common Runtime Errors
 
-### 1. Weight is negative, too large or not a valid number
+### Weight is negative, too large or not a valid number
 
 **Error:**
 ```
@@ -370,13 +745,12 @@ This joint processing allows cross-encoders to capture subtle semantic relations
 - Q3_K_M (3-bit) - Minimum recommended quantization
 
 ```ruby
-# Instead of Q2_K:
 llm = Candle::LLM.from_pretrained("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", 
                                   device: device, 
                                   gguf_file: "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf")
 ```
 
-### 2. Cannot find tensor model.embed_tokens.weight
+### Cannot find tensor model.embed_tokens.weight
 
 **Error:**
 ```
@@ -395,7 +769,7 @@ Failed to load quantized model: cannot find tensor model.embed_tokens.weight (Ru
    ```
 3. If the error persists, the GGUF file may use an unsupported architecture or format
 
-### 3. No GGUF file found in repository
+### No GGUF file found in repository
 
 **Error:**
 ```
@@ -412,7 +786,7 @@ llm = Candle::LLM.from_pretrained("TheBloke/Llama-2-7B-Chat-GGUF",
                                   gguf_file: "llama-2-7b-chat.Q4_K_M.gguf")
 ```
 
-### 4. Failed to download tokenizer
+### Failed to download tokenizer
 
 **Error:**
 ```
@@ -423,7 +797,7 @@ Failed to load quantized model: Failed to download tokenizer: request error: HTT
 
 **Solution:** The code now includes fallback tokenizer loading. If you still encounter this error, ensure you're using the latest version of red-candle.
 
-### 5. Missing metadata in GGUF file
+### Missing metadata in GGUF file
 
 **Error:**
 ```
@@ -452,17 +826,24 @@ Failed to load GGUF model: cannot find llama.attention.head_count in metadata (R
 FORK IT!
 
 ```
-git clone https://github.com/your_name/red-candle
+git clone https://github.com/assaydepot/red-candle
 cd red-candle
 bundle
 bundle exec rake compile
 ```
 
-Implemented with [Magnus](https://github.com/matsadler/magnus), with reference to [Polars Ruby](https://github.com/ankane/polars-ruby)
-
 Pull requests are welcome.
 
-### See Also
+## Release
+
+1. Update version number in `lib/candle/version.rb` and commit.
+2. `bundle exec rake build`
+3. `git tag VERSION_NUMBER`
+4. `git push --follow-tags`
+5. `gem push pkg/red-candle-1.0.0.gem`
+
+## See Also
 
-- [Numo::NArray](https://github.com/ruby-numo/numo-narray)
-- [Cumo](https://github.com/sonots/cumo)
+- [Candle](https://github.com/huggingface/candle)
+- [Magnus](https://github.com/matsadler/magnus)
+- [Outlines-core](https://github.com/dottxt-ai/outlines-core)

data/ext/candle/src/lib.rs

@@ -1,11 +1,13 @@
 use magnus::{function, prelude::*, Ruby};
 
 use crate::ruby::candle_utils;
-use crate::ruby::Result as RbResult;
+use crate::ruby::Result;
 
 pub mod llm;
+pub mod ner;
 pub mod reranker;
 pub mod ruby;
+pub mod tokenizer;
 
 // Configuration detection from build.rs
 #[cfg(all(has_metal, not(force_cpu)))]
@@ -33,7 +35,7 @@ pub fn get_build_info() -> magnus::RHash {
 }
 
 #[magnus::init]
-fn init(ruby: &Ruby) -> RbResult<()> {
+fn init(ruby: &Ruby) -> Result<()> {
     let rb_candle = ruby.define_module("Candle")?;
     
     // Export build info
@@ -41,11 +43,12 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     
     ruby::init_embedding_model(rb_candle)?;
     ruby::init_llm(rb_candle)?;
+    ner::init(rb_candle)?;
     reranker::init(rb_candle)?;
     ruby::dtype::init(rb_candle)?;
-    ruby::qtensor::init(rb_candle)?;
     ruby::device::init(rb_candle)?;
     ruby::tensor::init(rb_candle)?;
+    ruby::tokenizer::init(rb_candle)?;
     candle_utils(rb_candle)?;
 
     Ok(())

data/ext/candle/src/llm/gemma.rs

@@ -21,6 +21,11 @@ impl Gemma {
         self.model.clear_kv_cache();
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Gemma model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()

data/ext/candle/src/llm/llama.rs

@@ -28,6 +28,11 @@ impl Llama {
         }
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Llama model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()

data/ext/candle/src/llm/mistral.rs

@@ -21,6 +21,11 @@ impl Mistral {
         self.model.clear_kv_cache();
     }
     
+    /// Get the tokenizer
+    pub fn tokenizer(&self) -> &TokenizerWrapper {
+        &self.tokenizer
+    }
+    
     /// Load a Mistral model from HuggingFace Hub
     pub async fn from_pretrained(model_id: &str, device: Device) -> CandleResult<Self> {
         let api = Api::new()