red-candle 1.7.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d1b83311f0ad99adaffb886efda12291e9499ba112742e471e7b1eace390cee
-  data.tar.gz: 6156968c937204767fda21adfc6069e55a9571018f064ac56821d61472cc9cc1
+  metadata.gz: 2a4feba5be08ae2cb7a70b4e61671a5cd434c65a1fa29d98b77841187caf2893
+  data.tar.gz: 4c453e5cbf48837fa3b68af52f1f409ddaf039fe3f0b49e13a3c66693614f18f
 SHA512:
-  metadata.gz: 57abc7d285ebb3c67c438d861563247ec1130b03bee5fab418670bd926de58ad2bc06b2328732c362f46a4f8a8ccb7d85d7bd0e4b7a384838e290dadb7ed2be4
-  data.tar.gz: 2f10da2df023629d59b2c2454efa886a87e0738d99801e6bb25b951431856b083780167190a6ba735c3c51cb8d170ed16a9c163f19aaf44fc8b95a72f8ea4b5e
+  metadata.gz: 9c5b31367d0efd7d99ec866ab44615d94cbbd98cd401edb3d8393b892fdac0d1d3657a2903c02f60b20b2f8e3649a3eeac29d9f6348f3a0a51ec1fd97ad60757
+  data.tar.gz: 60da5b3c14e101b44e53321276c17e3a80074fb56e5e927469908ffb1deb17443b215c317405c526bb79a2d22df9e8837a32a3f7d486851b89e51a9cdc2ccf77

data/ext/candle/src/gvl.rs

@@ -0,0 +1,58 @@
+/// GVL (Global VM Lock) release support for Ruby.
+///
+/// Ruby's GVL prevents other Ruby threads from running while native code
+/// executes. For long-running operations (LLM inference, reranking, embedding),
+/// we release the GVL so other threads (TUI render loops, HTTP servers, etc.)
+/// can run concurrently.
+///
+/// SAFETY: Code running without the GVL must NOT call any Ruby API.
+
+use std::os::raw::c_void;
+
+type UnblockFn = unsafe extern "C" fn(*mut c_void);
+
+extern "C" {
+    fn rb_thread_call_without_gvl(
+        func: unsafe extern "C" fn(*mut c_void) -> *mut c_void,
+        data1: *mut c_void,
+        ubf: Option<UnblockFn>,
+        data2: *mut c_void,
+    ) -> *mut c_void;
+}
+
+/// Run a closure without the GVL. The closure must not call any Ruby API.
+pub fn without_gvl<F, R>(f: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    struct CallData<F, R> {
+        func: Option<F>,
+        result: Option<R>,
+    }
+
+    unsafe extern "C" fn call_func<F, R>(data: *mut c_void) -> *mut c_void
+    where
+        F: FnOnce() -> R,
+    {
+        let data = &mut *(data as *mut CallData<F, R>);
+        let func = data.func.take().unwrap();
+        data.result = Some(func());
+        std::ptr::null_mut()
+    }
+
+    let mut data = CallData {
+        func: Some(f),
+        result: None,
+    };
+
+    unsafe {
+        rb_thread_call_without_gvl(
+            call_func::<F, R>,
+            &mut data as *mut _ as *mut c_void,
+            None,
+            std::ptr::null_mut(),
+        );
+    }
+
+    data.result.unwrap()
+}

data/ext/candle/src/lib.rs

@@ -4,6 +4,7 @@ use crate::ruby::candle_utils;
 use crate::ruby::utils::ensure_hf_cache_dir;
 use crate::ruby::Result;
 
+pub mod gvl;
 pub mod llm;
 pub mod ruby;
 pub mod structured;

data/ext/candle/src/ruby/embedding_model.rs

@@ -104,14 +104,15 @@ impl EmbeddingModel {
     /// pooling_method: "pooled", "pooled_normalized", or "cls" (default: "pooled")
     pub fn embedding(&self, input: String, pooling_method: String) -> Result<Tensor> {
         let ruby = Ruby::get().unwrap();
-        match &self.0.model {
-            Some(model) => {
-                match &self.0.tokenizer {
-                    Some(tokenizer) => Ok(Tensor(self.compute_embedding(input, model, tokenizer, &pooling_method)?)),
-                    None => Err(magnus::Error::new(ruby.exception_runtime_error(), "Tokenizer not found"))
-                }
+        match (&self.0.model, &self.0.tokenizer) {
+            (Some(model), Some(tokenizer)) => {
+                let result = crate::gvl::without_gvl(|| {
+                    self.compute_embedding(input, model, tokenizer, &pooling_method)
+                });
+                Ok(Tensor(result?))
             }
-            None => Err(magnus::Error::new(ruby.exception_runtime_error(), "Model not found"))
+            (None, _) => Err(magnus::Error::new(ruby.exception_runtime_error(), "Model not found")),
+            (_, None) => Err(magnus::Error::new(ruby.exception_runtime_error(), "Tokenizer not found")),
         }
     }
 
@@ -119,14 +120,15 @@ impl EmbeddingModel {
     /// &RETURNS&: Tensor
     pub fn embeddings(&self, input: String) -> Result<Tensor> {
         let ruby = Ruby::get().unwrap();
-        match &self.0.model {
-            Some(model) => {
-                match &self.0.tokenizer {
-                    Some(tokenizer) => Ok(Tensor(self.compute_embeddings(input, model, tokenizer)?)),
-                    None => Err(magnus::Error::new(ruby.exception_runtime_error(), "Tokenizer not found"))
-                }
+        match (&self.0.model, &self.0.tokenizer) {
+            (Some(model), Some(tokenizer)) => {
+                let result = crate::gvl::without_gvl(|| {
+                    self.compute_embeddings(input, model, tokenizer)
+                });
+                Ok(Tensor(result?))
             }
-            None => Err(magnus::Error::new(ruby.exception_runtime_error(), "Model not found"))
+            (None, _) => Err(magnus::Error::new(ruby.exception_runtime_error(), "Model not found")),
+            (_, None) => Err(magnus::Error::new(ruby.exception_runtime_error(), "Tokenizer not found")),
         }
     }
 

data/ext/candle/src/ruby/llm.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 use crate::llm::{GenerationConfig as RustGenerationConfig, TextGenerator, mistral::Mistral as RustMistral, llama::Llama as RustLlama, gemma::Gemma as RustGemma, qwen::Qwen as RustQwen, qwen3::Qwen3 as RustQwen3, phi::Phi as RustPhi, granite::Granite as RustGranite, granitemoehybrid::GraniteMoeHybrid as RustGraniteMoeHybrid, glm4::Glm4 as RustGlm4, QuantizedGGUF as RustQuantizedGGUF};
 use crate::ruby::{Result, Device};
 use crate::ruby::structured::StructuredConstraint;
+use crate::gvl;
 
 // Use an enum to handle different model types instead of trait objects
 enum ModelType {
@@ -422,7 +423,7 @@ impl LLM {
         })
     }
 
-    /// Generate text from a prompt
+    /// Generate text from a prompt (releases GVL during inference)
     pub fn generate(&self, prompt: String, config: Option<&GenerationConfig>) -> Result<String> {
         let ruby = Ruby::get().unwrap();
         let config = config
@@ -435,8 +436,13 @@ impl LLM {
         };
         let mut model_ref = model.borrow_mut();
 
-        model_ref.generate(&prompt, &config)
-            .map_err(|e| Error::new(ruby.exception_runtime_error(), format!("Generation failed: {}", e)))
+        // Release the GVL during inference so other Ruby threads can run
+        // (e.g., TUI render loops, HTTP servers, etc.)
+        let result = gvl::without_gvl(|| {
+            model_ref.generate(&prompt, &config)
+        });
+
+        result.map_err(|e| Error::new(ruby.exception_runtime_error(), format!("Generation failed: {}", e)))
     }
 
     /// Generate text with streaming output

data/ext/candle/src/ruby/ner.rs

@@ -173,8 +173,10 @@ impl NER {
         let ruby = Ruby::get().unwrap();
         let threshold = confidence_threshold.unwrap_or(0.9) as f32;
 
-        // Use common tokenization and prediction logic
-        let (encoding, probs_vec) = self.tokenize_and_predict(&text)?;
+        // Release GVL during tokenization + model forward pass
+        let (encoding, probs_vec) = crate::gvl::without_gvl(|| {
+            self.tokenize_and_predict(&text)
+        })?;
 
         let tokens = encoding.get_tokens();
         let offsets = encoding.get_offsets();
@@ -208,8 +210,10 @@ impl NER {
     /// Get token-level predictions with labels and confidence scores
     pub fn predict_tokens(&self, text: String) -> Result<RArray> {
         let ruby = Ruby::get().unwrap();
-        // Use common tokenization and prediction logic
-        let (encoding, probs_vec) = self.tokenize_and_predict(&text)?;
+        // Release GVL during tokenization + model forward pass
+        let (encoding, probs_vec) = crate::gvl::without_gvl(|| {
+            self.tokenize_and_predict(&text)
+        })?;
 
         let tokens = encoding.get_tokens();
 

data/ext/candle/src/ruby/reranker.rs

@@ -18,6 +18,7 @@ use hf_hub::{api::sync::Api, Repo, RepoType};
 use tokenizers::{EncodeInput, Tokenizer};
 use std::cell::RefCell;
 use crate::ruby::{Device, Result};
+use crate::gvl;
 use crate::tokenizer::{TokenizerWrapper, loader::TokenizerLoader};
 
 enum RerankerModel {
@@ -164,39 +165,27 @@ impl Reranker {
     }
 
     /// Extract CLS embeddings from the model output, handling Metal device workarounds
-    fn extract_cls_embeddings(&self, embeddings: &Tensor) -> std::result::Result<Tensor, Error> {
-        let ruby = Ruby::get().unwrap();
-        let runtime_error = ruby.exception_runtime_error();
-
+    fn extract_cls_embeddings(&self, embeddings: &Tensor) -> std::result::Result<Tensor, String> {
         let cls_embeddings = if self.device.is_metal() {
-            // Metal has issues with tensor indexing, use a different approach
             let (batch_size, seq_len, hidden_size) = embeddings.dims3()
-                .map_err(|e| Error::new(runtime_error, format!("Failed to get dims: {}", e)))?;
-
-            // Reshape to [batch * seq_len, hidden] then take first hidden vectors for each batch
+                .map_err(|e| format!("Failed to get dims: {}", e))?;
             let reshaped = embeddings.reshape((batch_size * seq_len, hidden_size))
-                .map_err(|e| Error::new(runtime_error, format!("Failed to reshape: {}", e)))?;
-
-            // Extract CLS tokens (first token of each sequence)
+                .map_err(|e| format!("Failed to reshape: {}", e))?;
             let mut cls_vecs = Vec::new();
             for i in 0..batch_size {
                 let start_idx = i * seq_len;
                 let cls_vec = reshaped.narrow(0, start_idx, 1)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to extract CLS: {}", e)))?;
+                    .map_err(|e| format!("Failed to extract CLS: {}", e))?;
                 cls_vecs.push(cls_vec);
             }
-
-            // Stack the CLS vectors
             Tensor::cat(&cls_vecs, 0)
-                .map_err(|e| Error::new(runtime_error, format!("Failed to cat CLS tokens: {}", e)))?
+                .map_err(|e| format!("Failed to cat CLS tokens: {}", e))?
         } else {
             embeddings.i((.., 0))
-                .map_err(|e| Error::new(runtime_error, format!("Failed to extract CLS token: {}", e)))?
+                .map_err(|e| format!("Failed to extract CLS token: {}", e))?
         };
-
-        // Ensure tensor is contiguous for downstream operations
         cls_embeddings.contiguous()
-            .map_err(|e| Error::new(runtime_error, format!("Failed to make CLS embeddings contiguous: {}", e)))
+            .map_err(|e| format!("Failed to make CLS embeddings contiguous: {}", e))
     }
 
     pub fn debug_tokenization(&self, query: String, document: String) -> std::result::Result<RHash, Error> {
@@ -231,124 +220,147 @@ impl Reranker {
         let runtime_error = ruby.exception_runtime_error();
         let documents: Vec<String> = documents.to_vec()?;
 
+        // Release the GVL for the entire compute portion (tokenization + inference + scoring).
+        // None of this calls Ruby API.
+        let ranked_docs = gvl::without_gvl(|| -> std::result::Result<Vec<(String, f32, usize)>, String> {
+            self.compute_rerank(&query, &documents, &pooling_method, apply_sigmoid)
+        });
+
+        let ranked_docs = ranked_docs
+            .map_err(|e| Error::new(runtime_error, e))?;
+
+        // Build result array (requires GVL for Ruby object creation)
+        let result_array = ruby.ary_new();
+        for (doc, score, doc_id) in ranked_docs {
+            let tuple = ruby.ary_new();
+            tuple.push(doc)?;
+            tuple.push(ruby.float_from_f64(score as f64))?;
+            tuple.push(doc_id)?;
+            result_array.push(tuple)?;
+        }
+        Ok(result_array)
+    }
+
+    /// Pure compute portion of reranking — no Ruby API calls.
+    /// Returns ranked (document, score, original_index) tuples.
+    fn compute_rerank(&self, query: &str, documents: &[String], pooling_method: &str, apply_sigmoid: bool) -> std::result::Result<Vec<(String, f32, usize)>, String> {
         // Create query-document pairs for cross-encoder
         let query_and_docs: Vec<EncodeInput> = documents
             .iter()
-            .map(|d| (query.clone(), d.clone()).into())
+            .map(|d| (query.to_string(), d.clone()).into())
             .collect();
 
-        // Tokenize batch using inner tokenizer for access to token type IDs
+        // Tokenize batch
         let encodings = self.tokenizer.inner().encode_batch(query_and_docs, true)
-            .map_err(|e| Error::new(runtime_error, format!("Tokenization failed: {}", e)))?;
+            .map_err(|e| format!("Tokenization failed: {}", e))?;
 
-        // Convert to tensors
-        let token_ids = encodings
+        let token_ids_vec = encodings
             .iter()
             .map(|e| e.get_ids().to_vec())
             .collect::<Vec<_>>();
 
-        let token_type_ids = encodings
+        let token_type_ids_vec = encodings
             .iter()
             .map(|e| e.get_type_ids().to_vec())
             .collect::<Vec<_>>();
 
-        let token_ids = Tensor::new(token_ids, &self.device)
-            .map_err(|e| Error::new(runtime_error, format!("Failed to create tensor: {}", e)))?;
-        let token_type_ids = Tensor::new(token_type_ids, &self.device)
-            .map_err(|e| Error::new(runtime_error, format!("Failed to create token type ids tensor: {}", e)))?;
+        let token_ids = Tensor::new(token_ids_vec, &self.device)
+            .map_err(|e| format!("Failed to create tensor: {}", e))?;
+        let token_type_ids = Tensor::new(token_type_ids_vec, &self.device)
+            .map_err(|e| format!("Failed to create token type ids tensor: {}", e))?;
 
         // Compute scores based on model type
         let scores = match &self.model {
             RerankerModel::Bert { model, pooler, classifier } => {
                 let attention_mask = token_ids.ne(0u32)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to create attention mask: {}", e)))?;
+                    .map_err(|e| format!("Failed to create attention mask: {}", e))?;
 
                 // Forward pass through BERT
                 let embeddings = model.forward(&token_ids, &token_type_ids, Some(&attention_mask))
-                    .map_err(|e| Error::new(runtime_error, format!("Model forward pass failed: {}", e)))?;
+                    .map_err(|e| format!("Model forward pass failed: {}", e))?;
 
                 // Apply pooling based on the specified method
-                let pooled_embeddings = match pooling_method.as_str() {
+                let pooled_embeddings = match pooling_method {
                     "pooler" => {
                         let cls_embeddings = self.extract_cls_embeddings(&embeddings)?;
                         let pooled = pooler.forward(&cls_embeddings)
-                            .map_err(|e| Error::new(runtime_error, format!("Pooler forward failed: {}", e)))?;
+                            .map_err(|e| format!("Pooler forward failed: {}", e))?;
                         pooled.tanh()
-                            .map_err(|e| Error::new(runtime_error, format!("Tanh activation failed: {}", e)))?
+                            .map_err(|e| format!("Tanh activation failed: {}", e))?
                     },
                     "cls" => {
                         self.extract_cls_embeddings(&embeddings)?
                     },
                     "mean" => {
                         let (_batch, seq_len, _hidden) = embeddings.dims3()
-                            .map_err(|e| Error::new(runtime_error, format!("Failed to get tensor dimensions: {}", e)))?;
+                            .map_err(|e| format!("Failed to get tensor dimensions: {}", e))?;
                         let sum = embeddings.sum(1)
-                            .map_err(|e| Error::new(runtime_error, format!("Failed to sum embeddings: {}", e)))?;
+                            .map_err(|e| format!("Failed to sum embeddings: {}", e))?;
                         (sum / (seq_len as f64))
-                            .map_err(|e| Error::new(runtime_error, format!("Failed to compute mean: {}", e)))?
+                            .map_err(|e| format!("Failed to compute mean: {}", e))?
                     },
-                    _ => return Err(Error::new(runtime_error,
-                        format!("Unknown pooling method: {}. Use 'pooler', 'cls', or 'mean'", pooling_method)))
+                    _ => return Err(
+                        format!("Unknown pooling method: {}. Use 'pooler', 'cls', or 'mean'", pooling_method))
                 };
 
                 let pooled_embeddings = pooled_embeddings.contiguous()
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to make pooled_embeddings contiguous: {}", e)))?;
+                    .map_err(|e| format!("Failed to make pooled_embeddings contiguous: {}", e))?;
                 let logits = classifier.forward(&pooled_embeddings)
-                    .map_err(|e| Error::new(runtime_error, format!("Classifier forward failed: {}", e)))?;
+                    .map_err(|e| format!("Classifier forward failed: {}", e))?;
                 logits.squeeze(1)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to squeeze tensor: {}", e)))?
+                    .map_err(|e| format!("Failed to squeeze tensor: {}", e))?
             }
             RerankerModel::XLMRoberta { model, pad_token_id } => {
                 let attention_mask = token_ids.ne(*pad_token_id)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to create attention mask: {}", e)))?;
+                    .map_err(|e| format!("Failed to create attention mask: {}", e))?;
 
                 // XLMRobertaForSequenceClassification returns logits directly
                 let logits = model.forward(&token_ids, &attention_mask, &token_type_ids)
-                    .map_err(|e| Error::new(runtime_error, format!("Model forward pass failed: {}", e)))?;
+                    .map_err(|e| format!("Model forward pass failed: {}", e))?;
                 logits.squeeze(1)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to squeeze tensor: {}", e)))?
+                    .map_err(|e| format!("Failed to squeeze tensor: {}", e))?
             }
             RerankerModel::DeBERTa { model, pooler, classifier, pad_token_id } => {
                 let attention_mask = token_ids.ne(*pad_token_id)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to create attention mask: {}", e)))?;
+                    .map_err(|e| format!("Failed to create attention mask: {}", e))?;
 
                 // Forward through DeBERTa encoder
                 let encoder_output = model.forward(&token_ids, Some(token_type_ids.clone()), Some(attention_mask))
-                    .map_err(|e| Error::new(runtime_error, format!("Model forward pass failed: {}", e)))?;
+                    .map_err(|e| format!("Model forward pass failed: {}", e))?;
 
                 // Pool and classify
                 let pooled = pooler.forward(&encoder_output)
-                    .map_err(|e| Error::new(runtime_error, format!("Pooler forward failed: {}", e)))?;
+                    .map_err(|e| format!("Pooler forward failed: {}", e))?;
                 let logits = classifier.forward(&pooled)
-                    .map_err(|e| Error::new(runtime_error, format!("Classifier forward failed: {}", e)))?;
+                    .map_err(|e| format!("Classifier forward failed: {}", e))?;
                 logits.squeeze(1)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to squeeze tensor: {}", e)))?
+                    .map_err(|e| format!("Failed to squeeze tensor: {}", e))?
             }
             RerankerModel::ModernBert { model, head_dense, head_norm, classifier, pad_token_id } => {
                 let attention_mask = token_ids.ne(*pad_token_id)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to create attention mask: {}", e)))?;
+                    .map_err(|e| format!("Failed to create attention mask: {}", e))?;
                 let attention_mask_f32 = attention_mask.to_dtype(DType::F32)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to convert attention mask: {}", e)))?;
+                    .map_err(|e| format!("Failed to convert attention mask: {}", e))?;
 
                 // Forward through ModernBERT encoder
                 let encoder_output = model.forward(&token_ids, &attention_mask_f32)
-                    .map_err(|e| Error::new(runtime_error, format!("Model forward pass failed: {}", e)))?;
+                    .map_err(|e| format!("Model forward pass failed: {}", e))?;
 
                 // CLS pooling, then head (dense + GELU + norm) + classifier
                 let cls = encoder_output.i((.., 0, ..))
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to extract CLS: {}", e)))?
+                    .map_err(|e| format!("Failed to extract CLS: {}", e))?
                     .contiguous()
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to make contiguous: {}", e)))?;
+                    .map_err(|e| format!("Failed to make contiguous: {}", e))?;
                 let hidden = head_dense.forward(&cls)
-                    .map_err(|e| Error::new(runtime_error, format!("Head dense failed: {}", e)))?;
+                    .map_err(|e| format!("Head dense failed: {}", e))?;
                 let hidden = hidden.gelu_erf()
-                    .map_err(|e| Error::new(runtime_error, format!("GELU activation failed: {}", e)))?;
+                    .map_err(|e| format!("GELU activation failed: {}", e))?;
                 let hidden = head_norm.forward(&hidden)
-                    .map_err(|e| Error::new(runtime_error, format!("Head norm failed: {}", e)))?;
+                    .map_err(|e| format!("Head norm failed: {}", e))?;
                 let logits = classifier.forward(&hidden)
-                    .map_err(|e| Error::new(runtime_error, format!("Classifier forward failed: {}", e)))?;
+                    .map_err(|e| format!("Classifier forward failed: {}", e))?;
                 logits.squeeze(1)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to squeeze tensor: {}", e)))?
+                    .map_err(|e| format!("Failed to squeeze tensor: {}", e))?
             }
             RerankerModel::Qwen3 { model, yes_token_id, no_token_id } => {
                 // Qwen3 reranker: decoder-based yes/no scoring
@@ -356,7 +368,7 @@ impl Reranker {
                 let mut scores_vec: Vec<f32> = Vec::with_capacity(documents.len());
                 let mut model = model.borrow_mut();
 
-                for doc in &documents {
+                for doc in documents.iter() {
                     // Build the Qwen3 reranker prompt
                     let prompt = format!(
                         "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n<Instruct>: Given a web search query, retrieve relevant passages that answer the query\n<Query>: {}\n<Document>: {}<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
@@ -365,7 +377,7 @@ impl Reranker {
 
                     // Tokenize the prompt
                     let encoding = self.tokenizer.inner().encode(prompt.as_str(), false)
-                        .map_err(|e| Error::new(runtime_error, format!("Tokenization failed: {}", e)))?;
+                        .map_err(|e| format!("Tokenization failed: {}", e))?;
                     let input_ids: Vec<u32> = encoding.get_ids().to_vec();
 
                     // Clear KV cache for each document
@@ -373,28 +385,28 @@ impl Reranker {
 
                     // Forward pass — get logits for the last token position
                     let input_tensor = Tensor::new(&input_ids[..], &self.device)
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to create tensor: {}", e)))?
+                        .map_err(|e| format!("Failed to create tensor: {}", e))?
                         .unsqueeze(0)
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to unsqueeze: {}", e)))?;
+                        .map_err(|e| format!("Failed to unsqueeze: {}", e))?;
 
                     let logits = model.forward(&input_tensor, 0)
-                        .map_err(|e| Error::new(runtime_error, format!("Model forward pass failed: {}", e)))?;
+                        .map_err(|e| format!("Model forward pass failed: {}", e))?;
 
                     // logits shape: [1, 1, vocab_size] → flatten to [vocab_size]
                     let logits = logits.flatten_all()
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to flatten: {}", e)))?
+                        .map_err(|e| format!("Failed to flatten: {}", e))?
                         .to_dtype(DType::F32)
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to convert dtype: {}", e)))?;
+                        .map_err(|e| format!("Failed to convert dtype: {}", e))?;
 
                     // Extract yes/no logits and compute score
                     let yes_logit: f32 = logits.i(*yes_token_id as usize)
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to get yes logit: {}", e)))?
+                        .map_err(|e| format!("Failed to get yes logit: {}", e))?
                         .to_scalar()
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to convert yes logit: {}", e)))?;
+                        .map_err(|e| format!("Failed to convert yes logit: {}", e))?;
                     let no_logit: f32 = logits.i(*no_token_id as usize)
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to get no logit: {}", e)))?
+                        .map_err(|e| format!("Failed to get no logit: {}", e))?
                         .to_scalar()
-                        .map_err(|e| Error::new(runtime_error, format!("Failed to convert no logit: {}", e)))?;
+                        .map_err(|e| format!("Failed to convert no logit: {}", e))?;
 
                     // softmax over [yes, no] → P(yes)
                     let max_logit = yes_logit.max(no_logit);
@@ -407,24 +419,25 @@ impl Reranker {
 
                 // Build scores tensor for uniform handling below
                 Tensor::new(scores_vec.as_slice(), &self.device)
-                    .map_err(|e| Error::new(runtime_error, format!("Failed to create scores tensor: {}", e)))?
+                    .map_err(|e| format!("Failed to create scores tensor: {}", e))?
             }
         };
 
         // Optionally apply sigmoid activation
         let scores = if apply_sigmoid {
             sigmoid(&scores)
-                .map_err(|e| Error::new(runtime_error, format!("Sigmoid failed: {}", e)))?
+                .map_err(|e| format!("Sigmoid failed: {}", e))?
         } else {
             scores
         };
 
         let scores_vec: Vec<f32> = scores.to_vec1()
-            .map_err(|e| Error::new(runtime_error, format!("Failed to convert scores to vec: {}", e)))?;
+            .map_err(|e| format!("Failed to convert scores to vec: {}", e))?;
 
         // Create tuples with document, score, and original index
         let mut ranked_docs: Vec<(String, f32, usize)> = documents
-            .into_iter()
+            .iter()
+            .cloned()
             .zip(scores_vec)
             .enumerate()
             .map(|(idx, (doc, score))| (doc, score, idx))
@@ -433,16 +446,7 @@ impl Reranker {
         // Sort documents by relevance score (descending)
         ranked_docs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
 
-        // Build result array with [doc, score, doc_id]
-        let result_array = ruby.ary_new();
-        for (doc, score, doc_id) in ranked_docs {
-            let tuple = ruby.ary_new();
-            tuple.push(doc)?;
-            tuple.push(ruby.float_from_f64(score as f64))?;
-            tuple.push(doc_id)?;
-            result_array.push(tuple)?;
-        }
-        Ok(result_array)
+        Ok(ranked_docs)
     }
 
     /// Get the tokenizer used by this model

data/lib/candle/version.rb

@@ -1,5 +1,5 @@
 # :nocov:
 module Candle
-  VERSION = "1.7.0"
+  VERSION = "1.7.1"
 end
 # :nocov:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-candle
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Christopher Petersen
@@ -215,6 +215,7 @@ files:
 - ext/candle/build.rs
 - ext/candle/extconf.rb
 - ext/candle/rustfmt.toml
+- ext/candle/src/gvl.rs
 - ext/candle/src/lib.rs
 - ext/candle/src/llm/constrained_generation_test.rs
 - ext/candle/src/llm/gemma.rs