red-candle 1.8.0-aarch64-linux

data/ext/candle/src/tokenizer/mod.rs

@@ -0,0 +1,104 @@
+use candle_core::Result as CandleResult;
+use tokenizers::Tokenizer;
+
+pub mod loader;
+
+/// Common structure for managing tokenizer
+#[derive(Debug, Clone)]
+pub struct TokenizerWrapper {
+    tokenizer: Tokenizer,
+}
+
+impl TokenizerWrapper {
+    pub fn new(tokenizer: Tokenizer) -> Self {
+        Self { tokenizer }
+    }
+
+    pub fn encode(&self, text: &str, add_special_tokens: bool) -> CandleResult<Vec<u32>> {
+        let encoding = self.tokenizer
+            .encode(text, add_special_tokens)
+            .map_err(|e| candle_core::Error::Msg(format!("Tokenizer error: {}", e)))?;
+        Ok(encoding.get_ids().to_vec())
+    }
+
+    pub fn decode(&self, tokens: &[u32], skip_special_tokens: bool) -> CandleResult<String> {
+        self.tokenizer
+            .decode(tokens, skip_special_tokens)
+            .map_err(|e| candle_core::Error::Msg(format!("Tokenizer decode error: {}", e)))
+    }
+
+    pub fn token_to_piece(&self, token: u32) -> CandleResult<String> {
+        self.tokenizer
+            .id_to_token(token)
+            .map(|s| s.to_string())
+            .ok_or_else(|| candle_core::Error::Msg(format!("Unknown token id: {}", token)))
+    }
+    
+    /// Decode a single token for streaming output
+    pub fn decode_token(&self, token: u32) -> CandleResult<String> {
+        // Decode the single token properly
+        self.decode(&[token], true)
+    }
+    
+    /// Decode tokens incrementally for streaming
+    /// This is more efficient than decoding single tokens
+    pub fn decode_incremental(&self, all_tokens: &[u32], new_tokens_start: usize) -> CandleResult<String> {
+        if new_tokens_start >= all_tokens.len() {
+            return Ok(String::new());
+        }
+        
+        // Decode all tokens up to this point
+        let full_text = self.decode(all_tokens, true)?;
+        
+        // If we're at the start, return everything
+        if new_tokens_start == 0 {
+            return Ok(full_text);
+        }
+        
+        // Otherwise, decode up to the previous token and return the difference
+        let previous_text = self.decode(&all_tokens[..new_tokens_start], true)?;
+        
+        // Find the common prefix between the two strings to handle cases where
+        // the tokenizer might produce slightly different text when decoding
+        // different token sequences
+        let common_len = full_text
+            .chars()
+            .zip(previous_text.chars())
+            .take_while(|(a, b)| a == b)
+            .count();
+        
+        Ok(full_text.chars().skip(common_len).collect())
+    }
+    
+    /// Format tokens with debug information
+    pub fn format_tokens_with_debug(&self, tokens: &[u32]) -> CandleResult<String> {
+        let mut result = String::new();
+        for &token in tokens {
+            let piece = self.token_to_piece(token)?;
+            result.push_str(&format!("[{}:{}]", token, piece));
+        }
+        Ok(result)
+    }
+
+    /// Encode a batch of texts (needed for reranker)
+    pub fn encode_batch(&self, texts: Vec<String>, add_special_tokens: bool) -> CandleResult<Vec<Vec<u32>>> {
+        let encodings = self.tokenizer
+            .encode_batch(texts, add_special_tokens)
+            .map_err(|e| candle_core::Error::Msg(format!("Tokenizer batch error: {}", e)))?;
+        
+        Ok(encodings.into_iter()
+            .map(|encoding| encoding.get_ids().to_vec())
+            .collect())
+    }
+
+    /// Get the underlying tokenizer (for advanced use cases)
+    pub fn inner(&self) -> &Tokenizer {
+        &self.tokenizer
+    }
+
+    /// Get a mutable reference to the underlying tokenizer (for configuration)
+    pub fn inner_mut(&mut self) -> &mut Tokenizer {
+        &mut self.tokenizer
+    }
+}
+

data/ext/candle/tests/device_tests.rs

@@ -0,0 +1,43 @@
+use candle_core::Device as CoreDevice;
+
+#[test]
+fn test_device_creation() {
+    // CPU device should always work
+    let cpu = CoreDevice::Cpu;
+    assert!(matches!(cpu, CoreDevice::Cpu));
+    
+    // Test device display
+    assert_eq!(format!("{:?}", cpu), "Cpu");
+}
+
+#[cfg(feature = "cuda")]
+#[test]
+#[ignore = "requires CUDA hardware"]
+fn test_cuda_device_creation() {
+    // This might fail if no CUDA device is available
+    match CoreDevice::new_cuda(0) {
+        Ok(device) => assert!(matches!(device, CoreDevice::Cuda(_))),
+        Err(_) => println!("No CUDA device available for testing"),
+    }
+}
+
+#[cfg(feature = "metal")] 
+#[test]
+#[ignore = "requires Metal hardware"]
+fn test_metal_device_creation() {
+    // This might fail if no Metal device is available
+    match CoreDevice::new_metal(0) {
+        Ok(device) => assert!(matches!(device, CoreDevice::Metal(_))),
+        Err(_) => println!("No Metal device available for testing"),
+    }
+}
+
+#[test]
+fn test_device_matching() {
+    let cpu1 = CoreDevice::Cpu;
+    let cpu2 = CoreDevice::Cpu;
+    
+    // Same device types should match
+    assert!(matches!(cpu1, CoreDevice::Cpu));
+    assert!(matches!(cpu2, CoreDevice::Cpu));
+}

data/ext/candle/tests/tensor_tests.rs

@@ -0,0 +1,162 @@
+use candle_core::{Tensor, Device, DType};
+
+#[test]
+fn test_tensor_creation() {
+    let device = Device::Cpu;
+    
+    // Test tensor creation from slice
+    let data = vec![1.0f32, 2.0, 3.0, 4.0];
+    let tensor = Tensor::new(&data[..], &device).unwrap();
+    assert_eq!(tensor.dims(), &[4]);
+    assert_eq!(tensor.dtype(), DType::F32);
+    
+    // Test zeros
+    let zeros = Tensor::zeros(&[2, 3], DType::F32, &device).unwrap();
+    assert_eq!(zeros.dims(), &[2, 3]);
+    
+    // Test ones
+    let ones = Tensor::ones(&[3, 2], DType::F32, &device).unwrap();
+    assert_eq!(ones.dims(), &[3, 2]);
+}
+
+#[test]
+fn test_tensor_arithmetic() {
+    let device = Device::Cpu;
+    
+    let a = Tensor::new(&[1.0f32, 2.0, 3.0], &device).unwrap();
+    let b = Tensor::new(&[4.0f32, 5.0, 6.0], &device).unwrap();
+    
+    // Addition
+    let sum = a.add(&b).unwrap();
+    let sum_vec: Vec<f32> = sum.to_vec1().unwrap();
+    assert_eq!(sum_vec, vec![5.0, 7.0, 9.0]);
+    
+    // Subtraction
+    let diff = a.sub(&b).unwrap();
+    let diff_vec: Vec<f32> = diff.to_vec1().unwrap();
+    assert_eq!(diff_vec, vec![-3.0, -3.0, -3.0]);
+    
+    // Multiplication
+    let prod = a.mul(&b).unwrap();
+    let prod_vec: Vec<f32> = prod.to_vec1().unwrap();
+    assert_eq!(prod_vec, vec![4.0, 10.0, 18.0]);
+}
+
+#[test]
+fn test_tensor_reshape() {
+    let device = Device::Cpu;
+    
+    let tensor = Tensor::new(&[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0], &device).unwrap();
+    
+    // Reshape to 2x3
+    let reshaped = tensor.reshape(&[2, 3]).unwrap();
+    assert_eq!(reshaped.dims(), &[2, 3]);
+    
+    // Reshape to 3x2
+    let reshaped = tensor.reshape(&[3, 2]).unwrap();
+    assert_eq!(reshaped.dims(), &[3, 2]);
+}
+
+#[test]
+fn test_tensor_transpose() {
+    let device = Device::Cpu;
+    
+    let tensor = Tensor::new(&[1.0f32, 2.0, 3.0, 4.0], &device)
+        .unwrap()
+        .reshape(&[2, 2])
+        .unwrap();
+    
+    let transposed = tensor.transpose(0, 1).unwrap();
+    assert_eq!(transposed.dims(), &[2, 2]);
+    
+    let values: Vec<f32> = transposed.flatten_all().unwrap().to_vec1().unwrap();
+    assert_eq!(values, vec![1.0, 3.0, 2.0, 4.0]);
+}
+
+#[test]
+fn test_tensor_reduction() {
+    let device = Device::Cpu;
+    
+    let tensor = Tensor::new(&[1.0f32, 2.0, 3.0, 4.0], &device).unwrap();
+    
+    // Sum
+    let sum = tensor.sum_all().unwrap();
+    let sum_val: f32 = sum.to_scalar().unwrap();
+    assert_eq!(sum_val, 10.0);
+    
+    // Mean
+    let mean = tensor.mean_all().unwrap();
+    let mean_val: f32 = mean.to_scalar().unwrap();
+    assert_eq!(mean_val, 2.5);
+}
+
+#[test]
+fn test_tensor_indexing() {
+    let device = Device::Cpu;
+    
+    let tensor = Tensor::new(&[10.0f32, 20.0, 30.0, 40.0], &device).unwrap();
+    
+    // Get element at index 0
+    let elem = tensor.get(0).unwrap();
+    let val: f32 = elem.to_scalar().unwrap();
+    assert_eq!(val, 10.0);
+    
+    // Get element at index 2
+    let elem = tensor.get(2).unwrap();
+    let val: f32 = elem.to_scalar().unwrap();
+    assert_eq!(val, 30.0);
+}
+
+#[test]
+fn test_tensor_matmul() {
+    let device = Device::Cpu;
+    
+    // 2x3 matrix
+    let a = Tensor::new(&[1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0], &device)
+        .unwrap()
+        .reshape(&[2, 3])
+        .unwrap();
+    
+    // 3x2 matrix
+    let b = Tensor::new(&[7.0f32, 8.0, 9.0, 10.0, 11.0, 12.0], &device)
+        .unwrap()
+        .reshape(&[3, 2])
+        .unwrap();
+    
+    // Matrix multiplication
+    let result = a.matmul(&b).unwrap();
+    assert_eq!(result.dims(), &[2, 2]);
+    
+    let values: Vec<f32> = result.flatten_all().unwrap().to_vec1().unwrap();
+    // [1*7 + 2*9 + 3*11, 1*8 + 2*10 + 3*12, 4*7 + 5*9 + 6*11, 4*8 + 5*10 + 6*12]
+    // = [58, 64, 139, 154]
+    assert_eq!(values, vec![58.0, 64.0, 139.0, 154.0]);
+}
+
+#[test]
+fn test_tensor_where() {
+    let device = Device::Cpu;
+    
+    // Create a condition tensor where values > 0 are treated as true
+    let cond_values = Tensor::new(&[1.0f32, 0.0, 1.0], &device).unwrap();
+    let cond = cond_values.gt(&Tensor::zeros(cond_values.shape(), DType::F32, &device).unwrap()).unwrap();
+    
+    let on_true = Tensor::new(&[10.0f32, 20.0, 30.0], &device).unwrap();
+    let on_false = Tensor::new(&[100.0f32, 200.0, 300.0], &device).unwrap();
+    
+    let result = cond.where_cond(&on_true, &on_false).unwrap();
+    let values: Vec<f32> = result.to_vec1().unwrap();
+    assert_eq!(values, vec![10.0, 200.0, 30.0]);
+}
+
+#[test]
+fn test_tensor_narrow() {
+    let device = Device::Cpu;
+    
+    let tensor = Tensor::new(&[1.0f32, 2.0, 3.0, 4.0, 5.0], &device).unwrap();
+    
+    // Narrow from index 1, length 3
+    let narrowed = tensor.narrow(0, 1, 3).unwrap();
+    let values: Vec<f32> = narrowed.to_vec1().unwrap();
+    assert_eq!(values, vec![2.0, 3.0, 4.0]);
+}

data/lib/candle/agent.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Candle
+  class Agent
+    MAX_ITERATIONS = 10
+
+    attr_reader :llm, :tools, :system_prompt, :max_iterations
+
+    def initialize(llm, tools:, system_prompt: nil, max_iterations: MAX_ITERATIONS)
+      @llm = llm
+      @tools = tools
+      @system_prompt = system_prompt
+      @max_iterations = max_iterations
+    end
+
+    def run(user_message, **options)
+      messages = []
+      messages << { role: "system", content: @system_prompt } if @system_prompt
+      messages << { role: "user", content: user_message }
+
+      iterations = 0
+      loop do
+        iterations += 1
+        if iterations > @max_iterations
+          raise AgentMaxIterationsError,
+            "Agent exceeded maximum iterations (#{@max_iterations})"
+        end
+
+        result = @llm.chat_with_tools(messages, tools: @tools, execute: true, **options)
+
+        if result.has_tool_calls?
+          # If the model produced a substantial text answer alongside tool calls,
+          # treat it as a final response (model is done, trailing tool calls are noise).
+          # Strip <think> blocks so they don't count toward the length check.
+          text_without_thinking = result.text_response&.gsub(/<think>.*?<\/think>/m, "")&.strip
+          if text_without_thinking && text_without_thinking.length > 50
+            return AgentResult.new(
+              response: result.text_response,
+              messages: messages,
+              iterations: iterations,
+              tool_calls_made: messages.count { |m| m[:role] == "tool" }
+            )
+          end
+
+          messages << { role: "assistant", content: result.raw_response }
+
+          result.tool_results.each do |tr|
+            tool_name = tr[:tool_call]&.name || "unknown"
+            tool_output = tr[:error] ? "Error: #{tr[:error]}" : JSON.generate(tr[:result])
+            messages << { role: "tool", content: "[#{tool_name}] #{tool_output}" }
+          end
+        else
+          return AgentResult.new(
+            response: result.text_response || result.raw_response,
+            messages: messages,
+            iterations: iterations,
+            tool_calls_made: messages.count { |m| m[:role] == "tool" }
+          )
+        end
+      end
+    end
+  end
+
+  AgentResult = Struct.new(:response, :messages, :iterations, :tool_calls_made, keyword_init: true)
+  AgentMaxIterationsError = Class.new(StandardError)
+end

data/lib/candle/build_info.rb

@@ -0,0 +1,67 @@
+module Candle
+  module BuildInfo
+    def self.display_cuda_info
+      info = Candle.build_info
+      
+      # CUDA info is now controlled by logger level
+      
+      if info["cuda_available"] == false
+        # :nocov:
+        # Check if CUDA could be available on the system
+        cuda_potentially_available = ENV['CUDA_ROOT'] || ENV['CUDA_PATH'] || 
+                                   File.exist?('/usr/local/cuda') || File.exist?('/opt/cuda')
+        
+        if cuda_potentially_available
+          Candle.logger.warn "=" * 80
+          Candle.logger.warn "Red Candle: CUDA detected on system but not enabled in build."
+          Candle.logger.warn "This may be due to CANDLE_DISABLE_CUDA being set during installation."
+          Candle.logger.warn "To enable CUDA support, reinstall without CANDLE_DISABLE_CUDA set."
+          Candle.logger.warn "=" * 80
+        end
+        # :nocov:
+      end
+    end
+    
+    def self.cuda_available?
+      Candle.build_info["cuda_available"]
+    end
+
+    def self.metal_available?
+      Candle.build_info["metal_available"]
+    end
+
+    def self.mkl_available?
+      Candle.build_info["mkl_available"]
+    end
+
+    def self.accelerate_available?
+      Candle.build_info["accelerate_available"]
+    end
+
+    def self.cudnn_available?
+      Candle.build_info["cudnn_available"]
+    end
+
+    def self.summary
+      info = Candle.build_info
+      
+      available_backends = []
+      available_backends << "Metal" if info["metal_available"]
+      available_backends << "CUDA" if info["cuda_available"]
+      available_backends << "CPU"
+      
+      {
+        default_device: info["default_device"],
+        available_backends: available_backends,
+        cuda_available: info["cuda_available"],
+        metal_available: info["metal_available"],
+        mkl_available: info["mkl_available"],
+        accelerate_available: info["accelerate_available"],
+        cudnn_available: info["cudnn_available"]
+      }
+    end
+  end
+end
+
+# Display CUDA info on load
+Candle::BuildInfo.display_cuda_info

data/lib/candle/device_utils.rb

@@ -0,0 +1,10 @@
+module Candle
+  module DeviceUtils
+    # @deprecated Use {Candle::Device.best} instead
+    # Get the best available device (Metal > CUDA > CPU)
+    def self.best_device
+      Candle.logger.warn "[DEPRECATION] `DeviceUtils.best_device` is deprecated. Please use `Device.best` instead."
+      Device.best
+    end
+  end
+end

data/lib/candle/embedding_model.rb

@@ -0,0 +1,75 @@
+module Candle
+  class EmbeddingModel
+    # Default model path for Jina BERT embedding model
+    DEFAULT_MODEL_PATH = "jinaai/jina-embeddings-v2-base-en"
+    
+    # Default tokenizer path that works well with the default model
+    DEFAULT_TOKENIZER_PATH = "sentence-transformers/all-MiniLM-L6-v2"
+    
+    # Default embedding model type
+    DEFAULT_EMBEDDING_MODEL_TYPE = "jina_bert"
+    
+    # Load a pre-trained embedding model from HuggingFace
+    # @param model_id [String] HuggingFace model ID (defaults to jinaai/jina-embeddings-v2-base-en)
+    # @param device [Candle::Device] The device to use for computation (defaults to best available)
+    # @param tokenizer [String, nil] The tokenizer to use (defaults to using the model's tokenizer)
+    # @param model_type [String, nil] The type of embedding model (auto-detected if nil)
+    # @param embedding_size [Integer, nil] Override for the embedding size (optional)
+    # @return [EmbeddingModel] A new EmbeddingModel instance
+    def self.from_pretrained(model_id = DEFAULT_MODEL_PATH, device: Candle::Device.best, tokenizer: nil, model_type: nil, embedding_size: nil)
+      # Auto-detect model type based on model_id if not provided
+      if model_type.nil?
+        model_type = case model_id.downcase
+        when /jina/
+          "jina_bert"
+        when /distilbert/
+          "distilbert"
+        when /minilm/
+          "minilm"
+        else
+          "standard_bert"
+        end
+      end
+      
+      # Use model_id as tokenizer if not specified (usually what you want)
+      tokenizer_id = tokenizer || model_id
+      
+      _create(model_id, tokenizer_id, device, model_type, embedding_size)
+    end
+    
+    # Constructor for creating a new EmbeddingModel with optional parameters
+    # @deprecated Use {.from_pretrained} instead
+    # @param model_path [String, nil] The path to the model on Hugging Face
+    # @param tokenizer_path [String, nil] The path to the tokenizer on Hugging Face
+    # @param device [Candle::Device, Candle::Device.cpu] The device to use for computation
+    # @param model_type [String, nil] The type of embedding model to use
+    # @param embedding_size [Integer, nil] Override for the embedding size (optional)
+    def self.new(model_path: DEFAULT_MODEL_PATH,
+      tokenizer_path: DEFAULT_TOKENIZER_PATH,
+      device: Candle::Device.best,
+      model_type: DEFAULT_EMBEDDING_MODEL_TYPE,
+      embedding_size: nil)
+      $stderr.puts "[DEPRECATION] `EmbeddingModel.new` is deprecated. Please use `EmbeddingModel.from_pretrained` instead."
+      _create(model_path, tokenizer_path, device, model_type, embedding_size)
+    end
+    # Returns the embedding for a string using the specified pooling method.
+    # @param str [String] The input text
+    # @param pooling_method [String] Pooling method: "pooled", "pooled_normalized", or "cls". Default: "pooled_normalized"
+    def embedding(str, pooling_method: "pooled_normalized")
+      _embedding(str, pooling_method)
+    end
+    
+    # Improved inspect method
+    def inspect
+      opts = options rescue {}
+      
+      parts = ["#<Candle::EmbeddingModel"]
+      parts << "model=#{opts["model_id"] || "unknown"}"
+      parts << "type=#{opts["model_type"]}" if opts["model_type"]
+      parts << "device=#{opts["device"] || "unknown"}"
+      parts << "size=#{opts["embedding_size"]}" if opts["embedding_size"]
+      
+      parts.join(" ") + ">"
+    end
+  end
+end

data/lib/candle/embedding_model_type.rb

@@ -0,0 +1,31 @@
+module Candle
+  # Enum for the supported embedding model types
+  module EmbeddingModelType
+    # Jina Bert embedding models (e.g., jina-embeddings-v2-base-en)
+    JINA_BERT = "jina_bert"
+    
+    # Standard BERT embedding models (e.g., bert-base-uncased)
+    STANDARD_BERT = "standard_bert"
+    
+    # MiniLM embedding models (e.g., all-MiniLM-L6-v2)
+    MINILM = "minilm"
+    
+    # DistilBERT models which can be used for embeddings
+    DISTILBERT = "distilbert"
+    
+    # Returns a list of all supported model types
+    def self.all
+      [JINA_BERT, STANDARD_BERT, DISTILBERT, MINILM]
+    end
+    
+    # Returns suggested model paths for each model type
+    def self.suggested_model_paths
+      {
+        JINA_BERT => "jinaai/jina-embeddings-v2-base-en",
+        STANDARD_BERT => "scientistcom/BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext",
+        MINILM => "sentence-transformers/all-MiniLM-L6-v2",
+        DISTILBERT => "scientistcom/distilbert-base-uncased-finetuned-sst-2-english",
+      }
+    end
+  end
+end