red-candle 1.8.0.pre2-x86_64-linux

data/lib/candle/logger.rb

@@ -0,0 +1,149 @@
+require 'logger'
+
+module Candle
+  # Logging functionality for the Red Candle gem
+  class << self
+    # Get the current logger instance
+    # @return [Logger] The logger instance
+    def logger
+      @logger ||= create_default_logger
+    end
+
+    # Set a custom logger instance
+    # @param custom_logger [Logger] A custom logger instance
+    def logger=(custom_logger)
+      @logger = custom_logger
+    end
+
+    # Configure logging with a block
+    # @yield [config] Configuration object
+    def configure_logging
+      config = LoggerConfig.new
+      yield config if block_given?
+      @logger = config.build_logger
+    end
+
+    private
+
+    # Create the default logger with CLI-friendly settings
+    # @return [Logger] Configured logger instance
+    def create_default_logger
+      logger = Logger.new($stderr)
+      logger.level = default_log_level
+      logger.formatter = cli_friendly_formatter
+      logger
+    end
+
+    # Determine default log level based on environment variables
+    # @return [Integer] Logger level constant
+    def default_log_level
+      # Support legacy CANDLE_VERBOSE for backward compatibility, but prefer explicit configuration
+      return Logger::DEBUG if ENV['CANDLE_VERBOSE']
+      Logger::WARN  # CLI-friendly: only show warnings/errors by default
+    end
+
+    # CLI-friendly formatter that outputs just the message
+    # @return [Proc] Formatter proc
+    def cli_friendly_formatter
+      proc { |severity, datetime, progname, msg| "#{msg}\n" }
+    end
+  end
+
+  # Configuration helper for logger setup
+  class LoggerConfig
+    attr_accessor :level, :output, :formatter
+
+    def initialize
+      @level = :warn
+      @output = $stderr
+      @formatter = :simple
+    end
+
+    # Build a logger from the configuration
+    # @return [Logger] Configured logger
+    def build_logger
+      logger = Logger.new(@output)
+      logger.level = normalize_level(@level)
+      logger.formatter = build_formatter(@formatter)
+      logger
+    end
+
+    # Set log level to debug (verbose output)
+    def verbose!
+      @level = :debug
+    end
+
+    # Set log level to info  
+    def info!
+      @level = :info
+    end
+
+    # Set log level to warn (default)
+    def quiet!
+      @level = :warn
+    end
+
+    # Set log level to error (minimal output)
+    def silent!
+      @level = :error
+    end
+
+    # Log to stdout instead of stderr
+    def log_to_stdout!
+      @output = $stdout
+    end
+
+    # Log to a file
+    # @param file_path [String] Path to log file
+    def log_to_file!(file_path)
+      @output = file_path
+    end
+
+    # Disable logging completely
+    def disable!
+      @output = File::NULL
+    end
+
+    private
+
+    # Convert symbol/string level to Logger constant
+    # @param level [Symbol, String, Integer] Log level
+    # @return [Integer] Logger level constant
+    def normalize_level(level)
+      case level.to_s.downcase
+      when 'debug' then Logger::DEBUG
+      when 'info' then Logger::INFO
+      when 'warn', 'warning' then Logger::WARN
+      when 'error' then Logger::ERROR
+      when 'fatal' then Logger::FATAL
+      else Logger::WARN
+      end
+    end
+
+    # Build formatter based on type
+    # @param formatter_type [Symbol] Type of formatter
+    # @return [Proc] Formatter proc
+    def build_formatter(formatter_type)
+      case formatter_type
+      when :simple, :cli
+        proc { |severity, datetime, progname, msg| "#{msg}\n" }
+      when :detailed
+        proc do |severity, datetime, progname, msg|
+          "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity}: #{msg}\n"
+        end
+      when :json
+        require 'json'
+        proc do |severity, datetime, progname, msg|
+          JSON.generate({
+            timestamp: datetime.iso8601,
+            level: severity,
+            message: msg,
+            program: progname
+          }) + "\n"
+        end
+      else
+        proc { |severity, datetime, progname, msg| "#{msg}\n" }
+      end
+    end
+  end
+end

data/lib/candle/ner.rb

@@ -0,0 +1,368 @@
+# frozen_string_literal: true
+
+# Pattern validation available but not forced
+# require_relative 'pattern_validator'  # Uncomment if needed
+
+module Candle
+  # Named Entity Recognition (NER) for token classification
+  #
+  # This class provides methods to extract named entities from text using
+  # pre-trained BERT-based models. It supports standard NER labels like
+  # PER (person), ORG (organization), LOC (location), and can be extended
+  # with custom entity types.
+  #
+  # @example Load a pre-trained NER model
+  #   ner = Candle::NER.from_pretrained("Babelscape/wikineural-multilingual-ner")
+  #
+  # @example Load a model with a specific tokenizer
+  #   ner = Candle::NER.from_pretrained("dslim/bert-base-NER", tokenizer: "bert-base-cased")
+  #
+  # @example Extract entities from text
+  #   entities = ner.extract_entities("Apple Inc. was founded by Steve Jobs in Cupertino.")
+  #   # => [
+  #   #   { text: "Apple Inc.", label: "ORG", start: 0, end: 10, confidence: 0.99 },
+  #   #   { text: "Steve Jobs", label: "PER", start: 26, end: 36, confidence: 0.98 },
+  #   #   { text: "Cupertino", label: "LOC", start: 40, end: 49, confidence: 0.97 }
+  #   # ]
+  #
+  # @example Get token-level predictions
+  #   tokens = ner.predict_tokens("John works at Google")
+  #   # Returns detailed token-by-token predictions with confidence scores
+  class NER
+    class << self
+      # Load a pre-trained NER model from HuggingFace
+      #
+      # @param model_id [String] HuggingFace model ID (e.g., "dslim/bert-base-NER")
+      # @param device [Device] Device to run on (defaults to best available)
+      # @param tokenizer [String, nil] Tokenizer model ID to use (defaults to same as model_id)
+      # @return [NER] NER instance
+      def from_pretrained(model_id, device: Candle::Device.best, tokenizer: nil)
+        new(model_id, device, tokenizer)
+      end
+      
+      # Popular pre-trained models for different domains
+      def suggested_models
+        {
+          general: {
+            model: "Babelscape/wikineural-multilingual-ner",
+            note: "Has tokenizer.json"
+          },
+          general_alt: {
+            model: "dslim/bert-base-NER",
+            tokenizer: "bert-base-cased",
+            note: "Requires separate tokenizer"
+          },
+          multilingual: {
+            model: "Davlan/bert-base-multilingual-cased-ner-hrl",
+            note: "Check tokenizer availability"
+          },
+          biomedical: {
+            model: "dmis-lab/biobert-base-cased-v1.2",
+            note: "May require specific tokenizer"
+          },
+          clinical: {
+            model: "emilyalsentzer/Bio_ClinicalBERT",
+            note: "May require specific tokenizer"
+          },
+          scientific: {
+            model: "allenai/scibert_scivocab_uncased",
+            note: "May require specific tokenizer"
+          }
+        }
+      end
+    end
+    
+    # Create an alias for the native method
+    alias_method :_extract_entities, :extract_entities
+    
+    # Extract entities from text
+    #
+    # @param text [String] The text to analyze
+    # @param confidence_threshold [Float] Minimum confidence score (default: 0.9)
+    # @return [Array<Hash>] Array of entity hashes with text, label, start, end, confidence
+    def extract_entities(text, confidence_threshold: 0.9)
+      # Call the native method with positional arguments
+      _extract_entities(text, confidence_threshold)
+    end
+    
+    # Get available entity types
+    #
+    # @return [Array<String>] List of entity types (without B-/I- prefixes)
+    def entity_types
+      return @entity_types if @entity_types
+      
+      label_config = labels
+      @entity_types = label_config["label2id"].keys
+        .reject { |l| l == "O" }
+        .map { |l| l.sub(/^[BI]-/, "") }
+        .uniq
+        .sort
+    end
+    
+    # Check if model supports a specific entity type
+    #
+    # @param entity_type [String] Entity type to check (e.g., "GENE", "PER")
+    # @return [Boolean] Whether the model recognizes this entity type
+    def supports_entity?(entity_type)
+      entity_types.include?(entity_type.upcase)
+    end
+    
+    # Extract entities of a specific type
+    #
+    # @param text [String] The text to analyze
+    # @param entity_type [String] Entity type to extract (e.g., "PER", "ORG")
+    # @param confidence_threshold [Float] Minimum confidence score
+    # @return [Array<Hash>] Filtered entities of the specified type
+    def extract_entity_type(text, entity_type, confidence_threshold: 0.9)
+      entities = extract_entities(text, confidence_threshold: confidence_threshold)
+      entities.select { |e| e[:label] == entity_type.upcase }
+    end
+    
+    # Analyze text and return both entities and token predictions
+    #
+    # @param text [String] The text to analyze
+    # @param confidence_threshold [Float] Minimum confidence for entities
+    # @return [Hash] Hash with :entities and :tokens keys
+    def analyze(text, confidence_threshold: 0.9)
+      {
+        entities: extract_entities(text, confidence_threshold: confidence_threshold),
+        tokens: predict_tokens(text)
+      }
+    end
+    
+    # Get a formatted string representation of entities
+    #
+    # @param text [String] The text to analyze
+    # @param confidence_threshold [Float] Minimum confidence score
+    # @return [String] Formatted output with entities highlighted
+    def format_entities(text, confidence_threshold: 0.9)
+      entities = extract_entities(text, confidence_threshold: confidence_threshold)
+      return text if entities.empty?
+      
+      # Sort by start position (reverse for easier insertion)
+      entities.sort_by! { |e| -e[:start] }
+      
+      result = text.dup
+      entities.each do |entity|
+        label = "[#{entity[:label]}:#{entity[:confidence].round(2)}]"
+        result.insert(entity[:end], label)
+      end
+      
+      result
+    end
+    
+    # Get model information
+    #
+    # @return [String] Model description
+    def inspect
+      opts = options rescue {}
+      
+      parts = ["#<Candle::NER"]
+      parts << "model=#{opts["model_id"] || "unknown"}"
+      parts << "device=#{opts["device"] || "unknown"}"
+      parts << "labels=#{opts["num_labels"]}" if opts["num_labels"]
+      
+      if opts["entity_types"] && !opts["entity_types"].empty?
+        types = opts["entity_types"].sort.join(",")
+        parts << "types=#{types}"
+      end
+      
+      parts.join(" ") + ">"
+    end
+    
+    alias to_s inspect
+  end
+  
+  # Pattern-based entity recognizer for custom entities
+  class PatternEntityRecognizer
+    attr_reader :patterns, :entity_type
+    
+    def initialize(entity_type, patterns = [])
+      @entity_type = entity_type
+      @patterns = patterns
+    end
+    
+    # Add a pattern (String or Regexp)
+    def add_pattern(pattern)
+      @patterns << pattern
+      self
+    end
+    
+    # Recognize entities using patterns
+    def recognize(text, tokenizer = nil)
+      entities = []
+      
+      # Limit text length to prevent ReDoS on very long strings
+      # This is especially important for Ruby < 3.2
+      max_length = 1_000_000  # 1MB of text
+      if text.length > max_length
+        Candle.logger.warn "PatternEntityRecognizer: Text truncated from #{text.length} to #{max_length} chars for safety"
+        text = text[0...max_length]
+      end
+      
+      @patterns.each do |pattern|
+        regex = pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern)
+        
+        text.scan(regex) do |match|
+          match_text = $&
+          match_start = $~.offset(0)[0]
+          match_end = $~.offset(0)[1]
+          
+          entities << {
+            text: match_text,
+            label: @entity_type,
+            start: match_start,
+            end: match_end,
+            confidence: 1.0,
+            source: "pattern"
+          }
+        end
+      end
+      
+      entities
+    end
+  end
+  
+  # Gazetteer-based entity recognizer
+  class GazetteerEntityRecognizer
+    attr_reader :entity_type, :terms, :case_sensitive
+    
+    def initialize(entity_type, terms = [], case_sensitive: false)
+      @entity_type = entity_type
+      @case_sensitive = case_sensitive
+      @terms = build_term_set(terms)
+    end
+    
+    # Add terms to the gazetteer
+    def add_terms(terms)
+      terms = [terms] unless terms.is_a?(Array)
+      terms.each { |term| @terms.add(normalize_term(term)) }
+      self
+    end
+    
+    # Load terms from file
+    def load_from_file(filepath)
+      File.readlines(filepath).each do |line|
+        term = line.strip
+        add_terms(term) unless term.empty? || term.start_with?("#")
+      end
+      self
+    end
+    
+    # Recognize entities using the gazetteer
+    def recognize(text, tokenizer = nil)
+      entities = []
+      normalized_text = @case_sensitive ? text : text.downcase
+      
+      @terms.each do |term|
+        pattern = @case_sensitive ? term : term.downcase
+        pos = 0
+        
+        while (idx = normalized_text.index(pattern, pos))
+          # Check word boundaries
+          prev_char = idx > 0 ? text[idx - 1] : " "
+          next_char = idx + pattern.length < text.length ? text[idx + pattern.length] : " "
+          
+          if word_boundary?(prev_char) && word_boundary?(next_char)
+            entities << {
+              text: text[idx, pattern.length],
+              label: @entity_type,
+              start: idx,
+              end: idx + pattern.length,
+              confidence: 1.0,
+              source: "gazetteer"
+            }
+          end
+          
+          pos = idx + 1
+        end
+      end
+      
+      entities
+    end
+    
+    private
+    
+    def build_term_set(terms)
+      Set.new(terms.map { |term| normalize_term(term) })
+    end
+    
+    def normalize_term(term)
+      @case_sensitive ? term : term.downcase
+    end
+    
+    def word_boundary?(char)
+      char.match?(/\W/)
+    end
+  end
+  
+  # Hybrid NER that combines ML model with rules
+  class HybridNER
+    attr_reader :model_ner, :pattern_recognizers, :gazetteer_recognizers
+    
+    def initialize(model_id = nil, device: nil)
+      @model_ner = model_id ? NER.from_pretrained(model_id, device: device) : nil
+      @pattern_recognizers = []
+      @gazetteer_recognizers = []
+    end
+    
+    # Add a pattern-based recognizer
+    def add_pattern_recognizer(entity_type, patterns)
+      recognizer = PatternEntityRecognizer.new(entity_type, patterns)
+      @pattern_recognizers << recognizer
+      self
+    end
+    
+    # Add a gazetteer-based recognizer
+    def add_gazetteer_recognizer(entity_type, terms, **options)
+      recognizer = GazetteerEntityRecognizer.new(entity_type, terms, **options)
+      @gazetteer_recognizers << recognizer
+      self
+    end
+    
+    # Extract entities using all recognizers
+    def extract_entities(text, confidence_threshold: 0.9)
+      all_entities = []
+      
+      # Model-based entities
+      if @model_ner
+        model_entities = @model_ner.extract_entities(text, confidence_threshold: confidence_threshold)
+        all_entities.concat(model_entities)
+      end
+      
+      # Pattern-based entities
+      @pattern_recognizers.each do |recognizer|
+        pattern_entities = recognizer.recognize(text)
+        all_entities.concat(pattern_entities)
+      end
+      
+      # Gazetteer-based entities
+      @gazetteer_recognizers.each do |recognizer|
+        gazetteer_entities = recognizer.recognize(text)
+        all_entities.concat(gazetteer_entities)
+      end
+      
+      # Merge overlapping entities (prefer highest confidence)
+      merge_entities(all_entities)
+    end
+    
+    private
+    
+    def merge_entities(entities)
+      # Sort by start position and confidence (descending)
+      sorted = entities.sort_by { |e| [e[:start], -e[:confidence]] }
+      
+      merged = []
+      sorted.each do |entity|
+        # Check if entity overlaps with any already merged
+        overlaps = merged.any? do |existing|
+          entity[:start] < existing[:end] && entity[:end] > existing[:start]
+        end
+        
+        merged << entity unless overlaps
+      end
+      
+      merged.sort_by { |e| e[:start] }
+    end
+  end
+end

data/lib/candle/reranker.rb

@@ -0,0 +1,45 @@
+module Candle
+  class Reranker
+    # Default model path for cross-encoder/ms-marco-MiniLM-L-12-v2
+    DEFAULT_MODEL_PATH = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+    
+    # Load a pre-trained reranker model from HuggingFace
+    # @param model_id [String] HuggingFace model ID (defaults to cross-encoder/ms-marco-MiniLM-L-12-v2)
+    # @param device [Candle::Device] The device to use for computation (defaults to best available)
+    # @param max_length [Integer] Maximum sequence length for truncation (defaults to 512)
+    # @return [Reranker] A new Reranker instance
+    def self.from_pretrained(model_id = DEFAULT_MODEL_PATH, device: Candle::Device.best, max_length: 512)
+      _create(model_id, device, max_length)
+    end
+    
+    # Constructor for creating a new Reranker with optional parameters
+    # @deprecated Use {.from_pretrained} instead
+    # @param model_path [String, nil] The path to the model on Hugging Face
+    # @param device [Candle::Device, Candle::Device.cpu] The device to use for computation
+    # @param max_length [Integer] Maximum sequence length for truncation (defaults to 512)
+    def self.new(model_path: DEFAULT_MODEL_PATH, device: Candle::Device.best, max_length: 512)
+      $stderr.puts "[DEPRECATION] `Reranker.new` is deprecated. Please use `Reranker.from_pretrained` instead."
+      _create(model_path, device, max_length)
+    end
+
+    # Returns documents ranked by relevance using the specified pooling method.
+    # @param query [String] The input text
+    # @param documents [Array<String>] The list of documents to compare against
+    # @param pooling_method [String] Pooling method: "pooler", "cls", or "mean". Default: "pooler"
+    # @param apply_sigmoid [Boolean] Whether to apply sigmoid to the scores. Default: true
+    def rerank(query, documents, pooling_method: "pooler", apply_sigmoid: true)
+      rerank_with_options(query, documents, pooling_method, apply_sigmoid).collect { |doc, score, doc_id|
+        { doc_id: doc_id, score: score, text: doc }
+      }
+    end
+    
+    # Improved inspect method
+    def inspect
+      opts = options rescue {}
+      parts = ["#<Candle::Reranker"]
+      parts << "model=#{opts["model_id"] || "unknown"}"
+      parts << "device=#{opts["device"] || "unknown"}"
+      parts.join(" ") + ">"
+    end
+  end
+end

data/lib/candle/tensor.rb

@@ -0,0 +1,99 @@
+module Candle
+  class Tensor
+    include Enumerable
+
+    def each
+      case self.rank
+      when 0
+        # Scalar tensor - yield the single value
+        yield self.item
+      when 1
+        # 1D tensor - yield each value
+        # Check if we can use f32 values to avoid conversion
+        if dtype.to_s.downcase == "f32"
+          begin
+            values_f32.each { |value| yield value }
+          rescue NoMethodError
+            # :nocov:
+            # If values_f32 isn't available yet (not recompiled), fall back
+            if device.to_s != "cpu"
+              # Move to CPU to avoid Metal F32->F64 conversion issue
+              to_device(Candle::Device.cpu).values.each { |value| yield value }
+            else
+              values.each { |value| yield value }
+            end
+            # :nocov:
+          end
+        else
+          # For non-F32 dtypes, use regular values
+          values.each { |value| yield value }
+        end
+      else
+        # Multi-dimensional tensor - yield each sub-tensor
+        shape.first.times do |i|
+          yield self[i]
+        end
+      end
+    end
+    
+    # Convert scalar tensor to float
+    def to_f
+      if rank == 0
+        # Use item method which handles dtype conversion properly
+        item
+      else
+        raise ArgumentError, "to_f can only be called on scalar tensors (rank 0), but this tensor has rank #{rank}"
+      end
+    end
+    
+    # Convert scalar tensor to integer
+    def to_i
+      to_f.to_i
+    end
+    
+    # Improved inspect method showing shape, dtype, and device
+    def inspect
+      shape_str = shape.join("x")
+      
+      parts = ["#<Candle::Tensor"]
+      parts << "shape=#{shape_str}"
+      parts << "dtype=#{dtype}"
+      parts << "device=#{device}"
+      
+      # Add element count for clarity
+      parts << "elements=#{elem_count}"
+      
+      parts.join(" ") + ">"
+    end
+    
+    
+    # Override class methods to support keyword arguments for device
+    class << self
+      alias_method :_original_new, :new
+      alias_method :_original_ones, :ones
+      alias_method :_original_zeros, :zeros
+      alias_method :_original_rand, :rand
+      alias_method :_original_randn, :randn
+      
+      def new(data, dtype = nil, device: nil)
+        _original_new(data, dtype, device)
+      end
+      
+      def ones(shape, device: nil)
+        _original_ones(shape, device)
+      end
+      
+      def zeros(shape, device: nil)
+        _original_zeros(shape, device)
+      end
+      
+      def rand(shape, device: nil)
+        _original_rand(shape, device)
+      end
+      
+      def randn(shape, device: nil)
+        _original_randn(shape, device)
+      end
+    end
+  end
+end