grydra 1.0.0 → 2.0.1

data/lib/grydra/normalization.rb

@@ -0,0 +1,77 @@
+module GRYDRA
+  module Normalization
+    # Z-score normalization
+    def self.zscore_normalize(data)
+      n = data.size
+      means = data.first.size.times.map { |i| data.map { |row| row[i] }.sum.to_f / n }
+      std_devs = data.first.size.times.map do |i|
+        m = means[i]
+        Math.sqrt(data.map { |row| (row[i] - m)**2 }.sum.to_f / n)
+      end
+      normalized = data.map do |row|
+        row.each_with_index.map { |value, i| std_devs[i] != 0 ? (value - means[i]) / std_devs[i] : 0 }
+      end
+      [normalized, means, std_devs]
+    end
+
+    def self.zscore_denormalize(normalized, means, std_devs)
+      normalized.map do |row|
+        row.each_with_index.map { |value, i| value * std_devs[i] + means[i] }
+      end
+    end
+
+    # Min-Max normalization
+    def self.min_max_normalize(data, min_val = 0, max_val = 1)
+      data_min = data.flatten.min
+      data_max = data.flatten.max
+      range = data_max - data_min
+      return data if range == 0
+
+      data.map do |row|
+        row.map { |v| min_val + (v - data_min) * (max_val - min_val) / range }
+      end
+    end
+
+    # Generic normalization with multiple methods
+    def self.normalize_multiple(data, max_values, method = :max)
+      case method
+      when :max
+        data.map do |row|
+          row.each_with_index.map { |value, idx| value.to_f / max_values[idx] }
+        end
+      when :zscore
+        means = max_values[:means]
+        std_devs = max_values[:std_devs]
+        data.map do |row|
+          row.each_with_index.map do |value, idx|
+            std_devs[idx] != 0 ? (value.to_f - means[idx]) / std_devs[idx] : 0
+          end
+        end
+      else
+        raise ArgumentError, "Unknown normalization method: #{method}"
+      end
+    end
+
+    def self.calculate_max_values(data, method = :max)
+      if method == :max
+        max_values = {}
+        data.first.size.times do |i|
+          max_values[i] = data.map { |row| row[i] }.max.to_f
+        end
+        max_values
+      elsif method == :zscore
+        n = data.size
+        means = data.first.size.times.map do |i|
+          data.map { |row| row[i] }.sum.to_f / n
+        end
+        std_devs = data.first.size.times.map do |i|
+          m = means[i]
+          Math.sqrt(data.map { |row| (row[i] - m)**2 }.sum.to_f / n)
+        end
+        { means: means, std_devs: std_devs }
+      else
+        raise ArgumentError, "Unknown method for calculating max values: #{method}"
+      end
+    end
+  end
+end

data/lib/grydra/optimizers.rb

@@ -0,0 +1,162 @@
+module GRYDRA
+  module Optimizers
+    # Base Optimizer class
+    class Base
+      def update(parameter_id, gradient)
+        raise NotImplementedError, 'Subclasses must implement update method'
+      end
+
+      def reset
+        raise NotImplementedError, 'Subclasses must implement reset method'
+      end
+    end
+
+    # Stochastic Gradient Descent with Momentum
+    class SGD < Base
+      attr_reader :learning_rate, :momentum
+
+      def initialize(learning_rate: 0.01, momentum: 0.9, nesterov: false)
+        @learning_rate = learning_rate
+        @momentum = momentum
+        @nesterov = nesterov
+        @velocity = {}
+      end
+
+      def update(parameter_id, gradient)
+        @velocity[parameter_id] ||= 0
+        
+        if @nesterov
+          # Nesterov momentum
+          @velocity[parameter_id] = @momentum * @velocity[parameter_id] - @learning_rate * gradient
+          @momentum * @velocity[parameter_id] - @learning_rate * gradient
+        else
+          # Classical momentum
+          @velocity[parameter_id] = @momentum * @velocity[parameter_id] - @learning_rate * gradient
+          @velocity[parameter_id]
+        end
+      end
+
+      def reset
+        @velocity.clear
+      end
+    end
+
+    # Adam Optimizer (Adaptive Moment Estimation)
+    class Adam < Base
+      attr_reader :alpha, :beta1, :beta2, :epsilon
+
+      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, epsilon: 1e-8)
+        @alpha = alpha
+        @beta1 = beta1
+        @beta2 = beta2
+        @epsilon = epsilon
+        @m = {}
+        @v = {}
+        @t = 0
+      end
+
+      def update(parameter_id, gradient)
+        @t += 1
+        @m[parameter_id] ||= 0
+        @v[parameter_id] ||= 0
+
+        @m[parameter_id] = @beta1 * @m[parameter_id] + (1 - @beta1) * gradient
+        @v[parameter_id] = @beta2 * @v[parameter_id] + (1 - @beta2) * gradient**2
+
+        m_hat = @m[parameter_id] / (1 - @beta1**@t)
+        v_hat = @v[parameter_id] / (1 - @beta2**@t)
+
+        @alpha * m_hat / (Math.sqrt(v_hat) + @epsilon)
+      end
+
+      def reset
+        @m.clear
+        @v.clear
+        @t = 0
+      end
+    end
+
+    # RMSprop Optimizer
+    class RMSprop < Base
+      attr_reader :learning_rate, :decay_rate, :epsilon
+
+      def initialize(learning_rate: 0.001, decay_rate: 0.9, epsilon: 1e-8)
+        @learning_rate = learning_rate
+        @decay_rate = decay_rate
+        @epsilon = epsilon
+        @cache = {}
+      end
+
+      def update(parameter_id, gradient)
+        @cache[parameter_id] ||= 0
+        @cache[parameter_id] = @decay_rate * @cache[parameter_id] + (1 - @decay_rate) * gradient**2
+        @learning_rate * gradient / (Math.sqrt(@cache[parameter_id]) + @epsilon)
+      end
+
+      def reset
+        @cache.clear
+      end
+    end
+
+    # AdaGrad Optimizer
+    class AdaGrad < Base
+      attr_reader :learning_rate, :epsilon
+
+      def initialize(learning_rate: 0.01, epsilon: 1e-8)
+        @learning_rate = learning_rate
+        @epsilon = epsilon
+        @cache = {}
+      end
+
+      def update(parameter_id, gradient)
+        @cache[parameter_id] ||= 0
+        @cache[parameter_id] += gradient**2
+        @learning_rate * gradient / (Math.sqrt(@cache[parameter_id]) + @epsilon)
+      end
+
+      def reset
+        @cache.clear
+      end
+    end
+
+    # AdamW Optimizer (Adam with decoupled weight decay)
+    class AdamW < Base
+      attr_reader :alpha, :beta1, :beta2, :epsilon, :weight_decay
+
+      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, epsilon: 1e-8, weight_decay: 0.01)
+        @alpha = alpha
+        @beta1 = beta1
+        @beta2 = beta2
+        @epsilon = epsilon
+        @weight_decay = weight_decay
+        @m = {}
+        @v = {}
+        @t = 0
+      end
+
+      def update(parameter_id, gradient, parameter_value: 0)
+        @t += 1
+        @m[parameter_id] ||= 0
+        @v[parameter_id] ||= 0
+
+        @m[parameter_id] = @beta1 * @m[parameter_id] + (1 - @beta1) * gradient
+        @v[parameter_id] = @beta2 * @v[parameter_id] + (1 - @beta2) * gradient**2
+
+        m_hat = @m[parameter_id] / (1 - @beta1**@t)
+        v_hat = @v[parameter_id] / (1 - @beta2**@t)
+
+        # AdamW: decoupled weight decay
+        @alpha * (m_hat / (Math.sqrt(v_hat) + @epsilon) + @weight_decay * parameter_value)
+      end
+
+      def reset
+        @m.clear
+        @v.clear
+        @t = 0
+      end
+    end
+
+    # Alias for backward compatibility
+    AdamOptimizer = Adam
+  end
+end

data/lib/grydra/preprocessing/data.rb

@@ -0,0 +1,48 @@
+module GRYDRA
+  module Preprocessing
+    module Data
+      # Split data into training and test sets
+      def self.split_data(data_x, data_y, training_ratio = 0.8, seed = nil)
+        srand(seed) if seed
+        indices = (0...data_x.size).to_a.shuffle
+        cut = (data_x.size * training_ratio).to_i
+
+        {
+          train_x: indices[0...cut].map { |i| data_x[i] },
+          train_y: indices[0...cut].map { |i| data_y[i] },
+          test_x: indices[cut..-1].map { |i| data_x[i] },
+          test_y: indices[cut..-1].map { |i| data_y[i] }
+        }
+      end
+
+      # Generate synthetic data for testing
+      def self.generate_synthetic_data(n_samples, n_features, noise = 0.1, seed = nil)
+        srand(seed) if seed
+        data = Array.new(n_samples) do
+          Array.new(n_features) { rand * 2 - 1 + (rand * noise - noise / 2) }
+        end
+
+        # Generate labels based on a simple function
+        labels = data.map do |sample|
+          value = sample.each_with_index.sum { |x, i| x * (i + 1) * 0.1 }
+          [value + (rand * noise - noise / 2)]
+        end
+
+        { data: data, labels: labels }
+      end
+
+      # Convert hashes to vectors
+      def self.convert_hashes_to_vectors(array_hashes, keys)
+        array_hashes.map do |hash|
+          keys.map do |k|
+            if hash[k]
+              hash[k] == true ? 1.0 : (hash[k] == false ? 0.0 : hash[k].to_f)
+            else
+              0.0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/grydra/preprocessing/pca.rb

@@ -0,0 +1,132 @@
+module GRYDRA
+  module Preprocessing
+    module PCA
+      # Principal Component Analysis with Power Iteration
+      def self.pca(data, components: 2, max_iterations: 1000, tolerance: 1e-6)
+        n = data.size
+        m = data.first.size
+
+        # Center data
+        means = (0...m).map { |i| data.map { |row| row[i] }.sum.to_f / n }
+        centered_data = data.map { |row| row.zip(means).map { |v, mean| v - mean } }
+
+        # Calculate covariance matrix
+        covariance = calculate_covariance(centered_data, n, m)
+
+        # Find principal components using power iteration
+        principal_components = []
+        eigenvalues = []
+        
+        components.times do |comp_idx|
+          eigenvector, eigenvalue = power_iteration(
+            covariance, 
+            max_iterations: max_iterations, 
+            tolerance: tolerance,
+            deflate: principal_components
+          )
+          
+          break unless eigenvector
+          
+          principal_components << eigenvector
+          eigenvalues << eigenvalue
+        end
+
+        # Project data onto principal components
+        transformed_data = centered_data.map do |row|
+          principal_components.map do |pc|
+            row.zip(pc).map { |a, b| a * b }.sum
+          end
+        end
+
+        {
+          means: means,
+          principal_components: principal_components,
+          eigenvalues: eigenvalues,
+          explained_variance: calculate_explained_variance(eigenvalues),
+          transformed_data: transformed_data,
+          covariance: covariance
+        }
+      end
+
+      def self.transform(data, pca_result)
+        # Transform new data using existing PCA
+        centered = data.map do |row|
+          row.zip(pca_result[:means]).map { |v, mean| v - mean }
+        end
+
+        centered.map do |row|
+          pca_result[:principal_components].map do |pc|
+            row.zip(pc).map { |a, b| a * b }.sum
+          end
+        end
+      end
+
+      private
+
+      def self.calculate_covariance(centered_data, n, m)
+        covariance = Array.new(m) { Array.new(m, 0.0) }
+        (0...m).each do |i|
+          (0...m).each do |j|
+            covariance[i][j] = centered_data.map { |row| row[i] * row[j] }.sum / (n - 1).to_f
+          end
+        end
+        covariance
+      end
+
+      def self.power_iteration(matrix, max_iterations:, tolerance:, deflate: [])
+        n = matrix.size
+        
+        # Initialize random vector
+        vector = Array.new(n) { rand }
+        vector = normalize_vector(vector)
+
+        # Deflate for previously found components
+        deflate.each do |pc|
+          projection = vector.zip(pc).map { |a, b| a * b }.sum
+          vector = vector.zip(pc).map { |v, p| v - projection * p }
+        end
+        vector = normalize_vector(vector)
+
+        max_iterations.times do
+          # Multiply matrix by vector
+          new_vector = matrix_vector_multiply(matrix, vector)
+          
+          # Normalize
+          new_vector = normalize_vector(new_vector)
+          
+          # Check convergence
+          diff = vector.zip(new_vector).map { |a, b| (a - b).abs }.max
+          
+          vector = new_vector
+          
+          break if diff < tolerance
+        end
+
+        # Calculate eigenvalue (Rayleigh quotient)
+        numerator = matrix_vector_multiply(matrix, vector).zip(vector).map { |a, b| a * b }.sum
+        denominator = vector.map { |v| v**2 }.sum
+        eigenvalue = numerator / denominator
+
+        [vector, eigenvalue]
+      end
+
+      def self.matrix_vector_multiply(matrix, vector)
+        matrix.map do |row|
+          row.zip(vector).map { |a, b| a * b }.sum
+        end
+      end
+
+      def self.normalize_vector(vector)
+        magnitude = Math.sqrt(vector.map { |v| v**2 }.sum)
+        return vector if magnitude.zero?
+        vector.map { |v| v / magnitude }
+      end
+
+      def self.calculate_explained_variance(eigenvalues)
+        total = eigenvalues.sum
+        return [] if total.zero?
+        eigenvalues.map { |ev| ev / total }
+      end
+    end
+  end
+end

data/lib/grydra/preprocessing/text.rb

@@ -0,0 +1,62 @@
+module GRYDRA
+  module Preprocessing
+    module Text
+      # Create vocabulary from texts
+      def self.create_vocabulary(texts)
+        texts.map(&:split).flatten.map(&:downcase).uniq
+      end
+
+      # Create advanced vocabulary with frequency filtering
+      def self.create_advanced_vocabulary(texts, min_frequency = 1, max_words = nil)
+        frequencies = Hash.new(0)
+        texts.each do |text|
+          text.downcase.split.each { |word| frequencies[word] += 1 }
+        end
+
+        vocabulary = frequencies.select { |_, freq| freq >= min_frequency }.keys
+
+        if max_words && vocabulary.size > max_words
+          vocabulary = frequencies.sort_by { |_, freq| -freq }.first(max_words).map(&:first)
+        end
+
+        vocabulary.sort
+      end
+
+      # Vectorize text (binary)
+      def self.vectorize_text(text, vocabulary)
+        vector = Array.new(vocabulary.size, 0)
+        words = text.downcase.split
+        words.each do |word|
+          index = vocabulary.index(word)
+          vector[index] = 1 if index
+        end
+        vector
+      end
+
+      # Vectorize text using TF-IDF
+      def self.vectorize_text_tfidf(text, vocabulary, corpus_frequencies)
+        vector = Array.new(vocabulary.size, 0.0)
+        words = text.downcase.split
+        doc_frequencies = Hash.new(0)
+
+        words.each { |word| doc_frequencies[word] += 1 }
+
+        vocabulary.each_with_index do |word, idx|
+          next unless doc_frequencies[word] > 0
+
+          tf = doc_frequencies[word].to_f / words.size
+          idf = Math.log(corpus_frequencies.size.to_f / (corpus_frequencies[word] || 1))
+          vector[idx] = tf * idf
+        end
+
+        vector
+      end
+
+      # Normalize with vocabulary
+      def self.normalize_with_vocabulary(data, vocabulary)
+        max_value = vocabulary.size
+        data.map { |vector| vector.map { |v| v.to_f / max_value } }
+      end
+    end
+  end
+end

data/lib/grydra/regularization.rb

@@ -0,0 +1,19 @@
+module GRYDRA
+  module Regularization
+    # Apply dropout to outputs during training
+    def self.apply_dropout(outputs, dropout_rate = 0.5, training = true)
+      return outputs unless training
+      outputs.map { |s| rand < dropout_rate ? 0 : s / (1 - dropout_rate) }
+    end
+
+    # L1 regularization (Lasso)
+    def self.l1_regularization(weights, lambda_l1)
+      lambda_l1 * weights.sum { |p| p.abs }
+    end
+
+    # L2 regularization (Ridge)
+    def self.l2_regularization(weights, lambda_l2)
+      lambda_l2 * weights.sum { |p| p**2 }
+    end
+  end
+end

data/lib/grydra/training/cross_validation.rb

@@ -0,0 +1,35 @@
+module GRYDRA
+  module Training
+    module CrossValidation
+      # K-fold cross-validation
+      def self.cross_validation(data_input, data_output, k_folds = 5)
+        indices = (0...data_input.size).to_a.shuffle
+        fold_size = data_input.size / k_folds
+        errors = []
+
+        k_folds.times do |i|
+          start = i * fold_size
+          finish = [start + fold_size, data_input.size].min
+          indices_test = indices[start...finish]
+          indices_train = indices - indices_test
+
+          # Split data
+          train_x = indices_train.map { |idx| data_input[idx] }
+          train_y = indices_train.map { |idx| data_output[idx] }
+          test_x = indices_test.map { |idx| data_input[idx] }
+          test_y = indices_test.map { |idx| data_output[idx] }
+
+          # Train and evaluate
+          error = yield(train_x, train_y, test_x, test_y)
+          errors << error
+        end
+
+        {
+          errors: errors,
+          average: errors.sum / errors.size.to_f,
+          deviation: Math.sqrt(errors.map { |e| (e - errors.sum / errors.size.to_f)**2 }.sum / errors.size.to_f)
+        }
+      end
+    end
+  end
+end

data/lib/grydra/training/hyperparameter_search.rb

@@ -0,0 +1,46 @@
+module GRYDRA
+  module Training
+    module HyperparameterSearch
+      # Grid search for hyperparameters
+      def self.hyperparameter_search(data_x, data_y, param_grid, verbose: true)
+        best_params = nil
+        best_score = Float::INFINITY
+        results = []
+
+        puts 'Starting hyperparameter search...' if verbose
+
+        param_grid.each_with_index do |params, idx|
+          puts "Testing configuration #{idx + 1}/#{param_grid.size}: #{params}" if verbose
+
+          begin
+            score = yield(params, data_x, data_y)
+            results << { parameters: params, score: score }
+
+            if score < best_score
+              best_score = score
+              best_params = params
+              puts "  New best configuration! Score: #{score.round(6)}" if verbose
+            else
+              puts "  Score: #{score.round(6)}" if verbose
+            end
+          rescue StandardError => e
+            puts "  Error with this configuration: #{e.message}" if verbose
+            results << { parameters: params, score: Float::INFINITY, error: e.message }
+          end
+        end
+
+        if verbose
+          puts "\nBest parameters found:"
+          puts "  Configuration: #{best_params}"
+          puts "  Score: #{best_score.round(6)}"
+        end
+
+        {
+          parameters: best_params,
+          score: best_score,
+          all_results: results.sort_by { |r| r[:score] }
+        }
+      end
+    end
+  end
+end