red-candle 1.8.0.pre3-aarch64-linux

data/Rakefile

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/extensiontask"
+require "rspec/core/rake_task"
+
+task default: :spec
+
+spec = Bundler.load_gemspec("candle.gemspec")
+Rake::ExtensionTask.new("candle", spec) do |c|
+  c.lib_dir = "lib/candle"
+  c.cross_compile = true
+  c.cross_platform = %w[
+    aarch64-linux
+    arm64-darwin
+    x64-mingw-ucrt
+    x64-mingw32
+    x86_64-darwin
+    x86_64-linux
+    x86_64-linux-musl
+  ]
+end
+
+
+namespace :doc do
+  task default: %i[rustdoc yard]
+
+  desc "Generate YARD documentation"
+  task :yard do
+    sh <<~CMD
+      yard doc \
+        --plugin rustdoc -- lib tmp/doc/candle.json
+    CMD
+  end
+
+  desc "Generate Rust documentation as JSON"
+  task :rustdoc do
+    sh <<~CMD
+      cargo +nightly rustdoc \
+        --target-dir tmp/doc/target \
+        -p candle \
+        -- -Zunstable-options --output-format json \
+        --document-private-items
+    CMD
+
+    cp "tmp/doc/target/doc/candle.json", "tmp/doc/candle.json"
+  end
+end
+
+task doc: "doc:default"
+
+namespace :rust do
+  desc "Run Rust tests with code coverage"
+  namespace :coverage do
+    desc "Generate HTML coverage report"
+    task :html do
+      sh "cd ext/candle && cargo llvm-cov --html"
+      puts "Coverage report generated in target/llvm-cov/html/index.html"
+    end
+
+    desc "Generate coverage report in terminal"
+    task :report do
+      sh "cd ext/candle && cargo llvm-cov"
+    end
+
+    desc "Show coverage summary"
+    task :summary do
+      sh "cd ext/candle && cargo llvm-cov --summary-only"
+    end
+
+    desc "Generate lcov format coverage report"
+    task :lcov do
+      sh "cd ext/candle && cargo llvm-cov --lcov --output-path ../../coverage/lcov.info"
+      puts "LCOV report generated in coverage/lcov.info"
+    end
+
+    desc "Clean coverage data"
+    task :clean do
+      sh "cd ext/candle && cargo llvm-cov clean"
+    end
+  end
+
+  desc "Run Rust tests"
+  task :test do
+    sh "cd ext/candle && cargo test"
+  end
+end
+
+desc "Run Rust tests with coverage (alias)"
+task "coverage:rust" => "rust:coverage:html"
+
+# RSpec tasks
+desc "Run RSpec tests"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = "--format progress"
+end
+
+# Add compile as a dependency for spec task
+task spec: :compile
+
+namespace :spec do
+  desc "Run RSpec tests with all devices"
+  RSpec::Core::RakeTask.new(:device) do |t|
+    t.rspec_opts = "--format documentation --tag device"
+  end
+  
+  desc "Run RSpec tests with coverage"
+  task :coverage do
+    ENV['COVERAGE'] = 'true'
+    Rake::Task["spec"].invoke
+  end
+  
+  desc "Run RSpec tests in parallel (requires parallel_tests gem)"
+  task :parallel do
+    begin
+      require 'parallel_tests'
+      sh "parallel_rspec spec/"
+    rescue LoadError
+      puts "parallel_tests gem not installed. Run: gem install parallel_tests"
+    end
+  end
+  
+  desc "Run specific device tests"
+  %w[cpu metal cuda].each do |device|
+    desc "Run tests on #{device.upcase} only"
+    task "device:#{device}" => :compile do
+      ENV['CANDLE_TEST_DEVICES'] = device
+      sh "rspec spec/device_compatibility_spec.rb --format documentation"
+    end
+  end
+  
+  desc "Run LLM tests for specific models"
+  namespace :llm do
+    desc "Run tests for Gemma models"
+    task :gemma => :compile do
+      sh "rspec spec/llm/gemma_spec.rb --format documentation"
+    end
+    
+    desc "Run tests for Phi models"
+    task :phi => :compile do
+      sh "rspec spec/llm/phi_spec.rb --format documentation"
+    end
+    
+    desc "Run tests for Qwen models"
+    task :qwen => :compile do
+      sh "rspec spec/llm/qwen_spec.rb --format documentation"
+    end
+    
+    desc "Run tests for Mistral models"
+    task :mistral => :compile do
+      sh "rspec spec/llm/mistral_spec.rb --format documentation"
+    end
+    
+    desc "Run tests for Llama models"
+    task :llama => :compile do
+      sh "rspec spec/llm/llama_spec.rb --format documentation"
+    end
+
+    desc "Run tests for TinyLlama models"
+    task :tinyllama => :compile do
+      sh "rspec spec/llm/tinyllama_spec.rb --format documentation"
+    end
+
+    desc "Run all LLM tests (WARNING: requires large models already downloaded)"
+    task :all => [:gemma, :phi, :qwen, :mistral, :llama, :tinyllama]
+  end
+end

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "candle"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+# Check if Rust is installed
+if ! command -v cargo &> /dev/null
+then
+    echo "Rust is not installed. Please install Rust: https://www.rust-lang.org/tools/install"
+    exit 1
+fi
+
+# Install Ruby dependencies
+bundle install
+
+# Build Rust extension
+bundle exec rake compile

data/ext/candle/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "candle"
+version = "0.1.0"
+edition = "2021"
+build = "build.rs"
+rust-version = "1.85"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+candle-core = { version = "0.9.2" }
+candle-nn = { version = "0.9.2" }
+candle-transformers = { version = "0.9.2" }
+tokenizers = { version = "0.22.0", default-features = true, features = ["fancy-regex"] }
+# Drop hf-hub's default `default-tls` (= native-tls = openssl-sys), which can't
+# cross-compile to aarch64-linux (no OpenSSL headers in the cross sysroot). Keep
+# both API surfaces red-candle uses — `tokio` (async, via reqwest) and `ureq`
+# (sync) — and route reqwest through rustls (`rustls-tls`); ureq defaults to
+# rustls+ring. This removes openssl-sys from every platform's build.
+hf-hub = { version = "0.4.1", default-features = false, features = ["tokio", "ureq", "rustls-tls"] }
+half = "2.6.0"
+magnus = "0.8"
+safetensors = "0.3"
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+tokio = { version = "1.45", features = ["rt", "macros"] }
+rand = "0.10"
+outlines-core = "0.2.11"
+image = "0.25"
+
+[features]
+default = []
+metal = ["candle-core/metal", "candle-nn/metal", "candle-transformers/metal"]
+cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
+cudnn = ["candle-core/cudnn", "cuda"]
+mkl = ["candle-core/mkl"]
+accelerate = ["candle-core/accelerate"]

data/ext/candle/build.rs

@@ -0,0 +1,117 @@
+use std::env;
+use std::path::Path;
+
+fn main() {
+    // Register our custom cfg flags with rustc
+    println!("cargo::rustc-check-cfg=cfg(force_cpu)");
+    println!("cargo::rustc-check-cfg=cfg(has_cuda)");
+    println!("cargo::rustc-check-cfg=cfg(has_metal)");
+    println!("cargo::rustc-check-cfg=cfg(has_mkl)");
+    println!("cargo::rustc-check-cfg=cfg(has_accelerate)");
+    
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=CANDLE_FORCE_CPU");
+    println!("cargo:rerun-if-env-changed=CANDLE_CUDA_PATH");
+    println!("cargo:rerun-if-env-changed=CUDA_ROOT");
+    println!("cargo:rerun-if-env-changed=CUDA_PATH");
+    println!("cargo:rerun-if-env-changed=CANDLE_FEATURES");
+    println!("cargo:rerun-if-env-changed=CANDLE_ENABLE_CUDA");
+    println!("cargo:rerun-if-env-changed=CANDLE_DISABLE_CUDA");
+
+    // Check if we should force CPU only
+    if env::var("CANDLE_FORCE_CPU").is_ok() {
+        println!("cargo:rustc-cfg=force_cpu");
+        println!("cargo:warning=CANDLE_FORCE_CPU is set, disabling all acceleration");
+        return;
+    }
+
+    // Detect CUDA availability
+    let cuda_available = detect_cuda();
+    let cuda_disabled = env::var("CANDLE_DISABLE_CUDA").is_ok();
+    
+    if cuda_available && !cuda_disabled {
+        println!("cargo:rustc-cfg=has_cuda");
+        println!("cargo:warning=CUDA detected, CUDA acceleration will be available");
+    } else if cuda_available && cuda_disabled {
+        println!("cargo:warning=CUDA detected but disabled via CANDLE_DISABLE_CUDA");
+    }
+
+    // Detect Metal availability (macOS only)
+    #[cfg(target_os = "macos")]
+    {
+        println!("cargo:rustc-cfg=has_metal");
+        println!("cargo:warning=Metal detected (macOS), Metal acceleration will be available");
+    }
+
+    // Detect MKL availability
+    if detect_mkl() {
+        println!("cargo:rustc-cfg=has_mkl");
+        println!("cargo:warning=Intel MKL detected, MKL acceleration will be available");
+    }
+
+    // Detect Accelerate framework (macOS)
+    #[cfg(target_os = "macos")]
+    {
+        println!("cargo:rustc-cfg=has_accelerate");
+        println!("cargo:warning=Accelerate framework detected (macOS)");
+    }
+}
+
+fn detect_cuda() -> bool {
+    // Check environment variables first
+    if env::var("CANDLE_CUDA_PATH").is_ok() {
+        return true;
+    }
+
+    if env::var("CUDA_ROOT").is_ok() || env::var("CUDA_PATH").is_ok() {
+        return true;
+    }
+
+    // Check common CUDA installation paths
+    let cuda_paths = [
+        "/usr/local/cuda",
+        "/opt/cuda",
+        "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA",
+        "C:\\CUDA",
+    ];
+
+    for path in &cuda_paths {
+        if Path::new(path).exists() {
+            return true;
+        }
+    }
+
+    // Check if nvcc is in PATH
+    if let Ok(path_var) = env::var("PATH") {
+        for path in env::split_paths(&path_var) {
+            if path.join("nvcc").exists() || path.join("nvcc.exe").exists() {
+                return true;
+            }
+        }
+    }
+
+    false
+}
+
+fn detect_mkl() -> bool {
+    // Check environment variables
+    if env::var("MKLROOT").is_ok() || env::var("MKL_ROOT").is_ok() {
+        return true;
+    }
+
+    // Check common MKL installation paths
+    let mkl_paths = [
+        "/opt/intel/mkl",
+        "/opt/intel/oneapi/mkl/latest",
+        "C:\\Program Files (x86)\\Intel\\oneAPI\\mkl\\latest",
+        "C:\\Program Files\\Intel\\oneAPI\\mkl\\latest",
+    ];
+
+    for path in &mkl_paths {
+        if Path::new(path).exists() {
+            return true;
+        }
+    }
+
+    false
+}

data/ext/candle/extconf.rb

@@ -0,0 +1,79 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+# Detect available hardware acceleration
+features = []
+
+# Force CPU-only build if requested
+if ENV['CANDLE_FORCE_CPU']
+  puts "CANDLE_FORCE_CPU is set, building CPU-only version"
+else
+  # Check for CUDA
+  cuda_available = ENV['CUDA_ROOT'] || ENV['CUDA_PATH'] || ENV['CANDLE_CUDA_PATH'] ||
+                   File.exist?('/usr/local/cuda') || File.exist?('/opt/cuda') ||
+                   (RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ && 
+                    (File.exist?('C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA') ||
+                     File.exist?('C:\CUDA')))
+  
+  cuda_disabled = ENV['CANDLE_DISABLE_CUDA']
+  
+  if cuda_available && !cuda_disabled
+    puts "CUDA detected, enabling CUDA support"
+    features << 'cuda'
+    
+    # Check if CUDNN should be enabled
+    if ENV['CANDLE_CUDNN'] || ENV['CUDNN_ROOT']
+      puts "CUDNN support enabled"
+      features << 'cudnn'
+    end
+  elsif cuda_available && cuda_disabled
+    puts "=" * 80
+    puts "CUDA detected but disabled via CANDLE_DISABLE_CUDA"
+    puts "=" * 80
+  end
+
+  # Check for Metal (macOS only)
+  if RbConfig::CONFIG['host_os'] =~ /darwin/
+    puts "macOS detected, enabling Metal support"
+    features << 'metal'
+    
+    # Also enable Accelerate framework on macOS
+    puts "Enabling Accelerate framework support"
+    features << 'accelerate'
+  end
+
+  # Check for Intel MKL
+  mkl_available = ENV['MKLROOT'] || ENV['MKL_ROOT'] ||
+                  File.exist?('/opt/intel/mkl') || 
+                  File.exist?('/opt/intel/oneapi/mkl/latest')
+  
+  if mkl_available && !features.include?('accelerate')  # Don't use both MKL and Accelerate
+    puts "Intel MKL detected, enabling MKL support"
+    features << 'mkl'
+  end
+end
+
+# Allow manual override of features
+if ENV['CANDLE_FEATURES']
+  manual_features = ENV['CANDLE_FEATURES'].split(',').map(&:strip)
+  puts "Manual features override: #{manual_features.join(', ')}"
+  features = manual_features
+end
+
+# Display selected features
+unless features.empty?
+  puts "Building with features: #{features.join(', ')}"
+else
+  puts "Building CPU-only version (no acceleration features detected)"
+end
+
+# Create the Rust makefile with proper feature configuration
+create_rust_makefile("candle/candle") do |r|
+  # Pass the features to rb_sys
+  r.features = features unless features.empty?
+  
+  # Pass through any additional cargo flags
+  if ENV['CANDLE_CARGO_FLAGS']
+    r.extra_cargo_args = ENV['CANDLE_CARGO_FLAGS'].split(' ')
+  end
+end

data/ext/candle/rustfmt.toml

@@ -0,0 +1,63 @@
+max_width = 100
+hard_tabs = false
+tab_spaces = 4
+newline_style = "Auto"
+use_small_heuristics = "Default"
+indent_style = "Block"
+wrap_comments = false
+format_code_in_doc_comments = false
+comment_width = 80
+normalize_comments = false
+normalize_doc_attributes = false
+format_strings = false
+format_macro_matchers = false
+format_macro_bodies = true
+empty_item_single_line = true
+struct_lit_single_line = true
+fn_single_line = false
+where_single_line = false
+imports_indent = "Block"
+imports_layout = "Mixed"
+imports_granularity="Crate"
+reorder_imports = true
+reorder_modules = true
+reorder_impl_items = false
+type_punctuation_density = "Wide"
+space_before_colon = false
+space_after_colon = true
+spaces_around_ranges = false
+binop_separator = "Front"
+remove_nested_parens = true
+combine_control_expr = true
+overflow_delimited_expr = false
+struct_field_align_threshold = 0
+enum_discrim_align_threshold = 0
+match_arm_blocks = true
+force_multiline_blocks = false
+fn_params_layout = "Tall"
+brace_style = "SameLineWhere"
+control_brace_style = "AlwaysSameLine"
+trailing_semicolon = true
+trailing_comma = "Vertical"
+match_block_trailing_comma = false
+blank_lines_upper_bound = 1
+blank_lines_lower_bound = 0
+edition = "2021"
+version = "One"
+inline_attribute_width = 0
+merge_derives = true
+use_try_shorthand = false
+use_field_init_shorthand = false
+force_explicit_abi = true
+condense_wildcard_suffixes = false
+color = "Auto"
+#required_version = "1.4.12"
+unstable_features = false
+disable_all_formatting = false
+skip_children = false
+hide_parse_errors = false
+error_on_line_overflow = false
+error_on_unformatted = false
+ignore = []
+emit_mode = "Files"
+make_backup = false

data/ext/candle/src/gvl.rs

@@ -0,0 +1,58 @@
+/// GVL (Global VM Lock) release support for Ruby.
+///
+/// Ruby's GVL prevents other Ruby threads from running while native code
+/// executes. For long-running operations (LLM inference, reranking, embedding),
+/// we release the GVL so other threads (TUI render loops, HTTP servers, etc.)
+/// can run concurrently.
+///
+/// SAFETY: Code running without the GVL must NOT call any Ruby API.
+
+use std::os::raw::c_void;
+
+type UnblockFn = unsafe extern "C" fn(*mut c_void);
+
+extern "C" {
+    fn rb_thread_call_without_gvl(
+        func: unsafe extern "C" fn(*mut c_void) -> *mut c_void,
+        data1: *mut c_void,
+        ubf: Option<UnblockFn>,
+        data2: *mut c_void,
+    ) -> *mut c_void;
+}
+
+/// Run a closure without the GVL. The closure must not call any Ruby API.
+pub fn without_gvl<F, R>(f: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    struct CallData<F, R> {
+        func: Option<F>,
+        result: Option<R>,
+    }
+
+    unsafe extern "C" fn call_func<F, R>(data: *mut c_void) -> *mut c_void
+    where
+        F: FnOnce() -> R,
+    {
+        let data = &mut *(data as *mut CallData<F, R>);
+        let func = data.func.take().unwrap();
+        data.result = Some(func());
+        std::ptr::null_mut()
+    }
+
+    let mut data = CallData {
+        func: Some(f),
+        result: None,
+    };
+
+    unsafe {
+        rb_thread_call_without_gvl(
+            call_func::<F, R>,
+            &mut data as *mut _ as *mut c_void,
+            None,
+            std::ptr::null_mut(),
+        );
+    }
+
+    data.result.unwrap()
+}

data/ext/candle/src/lib.rs

@@ -0,0 +1,59 @@
+use magnus::{function, prelude::*, Ruby};
+
+use crate::ruby::candle_utils;
+use crate::ruby::utils::ensure_hf_cache_dir;
+use crate::ruby::Result;
+
+pub mod gvl;
+pub mod llm;
+pub mod ruby;
+pub mod structured;
+pub mod tokenizer;
+
+// Configuration detection from build.rs
+#[cfg(all(has_metal, not(force_cpu)))]
+const DEFAULT_DEVICE: &str = "metal";
+
+#[cfg(all(has_cuda, not(has_metal), not(force_cpu)))]
+const DEFAULT_DEVICE: &str = "cuda";
+
+#[cfg(any(force_cpu, not(any(has_metal, has_cuda))))]
+const DEFAULT_DEVICE: &str = "cpu";
+
+// Export build configuration for runtime checks
+pub fn get_build_info() -> magnus::RHash {
+    let ruby = magnus::Ruby::get().unwrap();
+    let hash = ruby.hash_new();
+
+    let _ = hash.aset("default_device", DEFAULT_DEVICE);
+    let _ = hash.aset("cuda_available", cfg!(feature = "cuda"));
+    let _ = hash.aset("metal_available", cfg!(feature = "metal"));
+    let _ = hash.aset("mkl_available", cfg!(feature = "mkl"));
+    let _ = hash.aset("accelerate_available", cfg!(feature = "accelerate"));
+    let _ = hash.aset("cudnn_available", cfg!(feature = "cudnn"));
+
+    hash
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<()> {
+    ensure_hf_cache_dir();
+    let rb_candle = ruby.define_module("Candle")?;
+    
+    // Export build info
+    rb_candle.define_singleton_method("build_info", function!(get_build_info, 0))?;
+    
+    ruby::init_embedding_model(rb_candle)?;
+    ruby::init_llm(rb_candle)?;
+    ruby::ner::init(rb_candle)?;
+    ruby::reranker::init(rb_candle)?;
+    ruby::vlm::init(rb_candle)?;
+    ruby::dtype::init(rb_candle)?;
+    ruby::device::init(rb_candle)?;
+    ruby::tensor::init(rb_candle)?;
+    ruby::tokenizer::init(rb_candle)?;
+    ruby::structured::init_structured(rb_candle)?;
+    candle_utils(rb_candle)?;
+
+    Ok(())
+}