grydra 1.0.0 → 2.0.1

data/lib/grydra/losses.rb

@@ -0,0 +1,119 @@
+module GRYDRA
+  module Losses
+    # Mean Squared Error Loss
+    def self.mse(predictions, targets)
+      n = predictions.size
+      sum = predictions.zip(targets).map { |p, t| (p - t)**2 }.sum
+      sum / n.to_f
+    end
+
+    def self.mse_derivative(predictions, targets)
+      predictions.zip(targets).map { |p, t| 2 * (p - t) / predictions.size }
+    end
+
+    # Mean Absolute Error Loss
+    def self.mae(predictions, targets)
+      n = predictions.size
+      sum = predictions.zip(targets).map { |p, t| (p - t).abs }.sum
+      sum / n.to_f
+    end
+
+    def self.mae_derivative(predictions, targets)
+      predictions.zip(targets).map { |p, t| p > t ? 1.0 / predictions.size : -1.0 / predictions.size }
+    end
+
+    # Binary Cross-Entropy Loss
+    def self.binary_crossentropy(predictions, targets, epsilon: 1e-7)
+      predictions = predictions.map { |p| [[p, epsilon].max, 1 - epsilon].min }
+      n = predictions.size
+      sum = predictions.zip(targets).map do |p, t|
+        -(t * Math.log(p) + (1 - t) * Math.log(1 - p))
+      end.sum
+      sum / n.to_f
+    end
+
+    def self.binary_crossentropy_derivative(predictions, targets, epsilon: 1e-7)
+      predictions = predictions.map { |p| [[p, epsilon].max, 1 - epsilon].min }
+      predictions.zip(targets).map do |p, t|
+        (p - t) / (p * (1 - p) * predictions.size)
+      end
+    end
+
+    # Categorical Cross-Entropy Loss
+    def self.categorical_crossentropy(predictions, targets, epsilon: 1e-7)
+      predictions = predictions.map { |p| [p, epsilon].max }
+      n = predictions.size
+      sum = predictions.zip(targets).map do |pred_row, target_row|
+        pred_row.zip(target_row).map { |p, t| -t * Math.log(p) }.sum
+      end.sum
+      sum / n.to_f
+    end
+
+    # Huber Loss (robust to outliers)
+    def self.huber(predictions, targets, delta: 1.0)
+      n = predictions.size
+      sum = predictions.zip(targets).map do |p, t|
+        error = (p - t).abs
+        if error <= delta
+          0.5 * error**2
+        else
+          delta * (error - 0.5 * delta)
+        end
+      end.sum
+      sum / n.to_f
+    end
+
+    def self.huber_derivative(predictions, targets, delta: 1.0)
+      predictions.zip(targets).map do |p, t|
+        error = p - t
+        if error.abs <= delta
+          error / predictions.size
+        else
+          delta * (error > 0 ? 1 : -1) / predictions.size
+        end
+      end
+    end
+
+    # Hinge Loss (for SVM-style classification)
+    def self.hinge(predictions, targets)
+      n = predictions.size
+      sum = predictions.zip(targets).map do |p, t|
+        [0, 1 - t * p].max
+      end.sum
+      sum / n.to_f
+    end
+
+    def self.hinge_derivative(predictions, targets)
+      predictions.zip(targets).map do |p, t|
+        (1 - t * p) > 0 ? -t / predictions.size : 0
+      end
+    end
+
+    # Log-Cosh Loss (smooth approximation of MAE)
+    def self.log_cosh(predictions, targets)
+      n = predictions.size
+      sum = predictions.zip(targets).map do |p, t|
+        x = p - t
+        Math.log(Math.cosh(x))
+      end.sum
+      sum / n.to_f
+    end
+
+    def self.log_cosh_derivative(predictions, targets)
+      predictions.zip(targets).map do |p, t|
+        x = p - t
+        Math.tanh(x) / predictions.size
+      end
+    end
+
+    # Quantile Loss (for quantile regression)
+    def self.quantile(predictions, targets, quantile: 0.5)
+      n = predictions.size
+      sum = predictions.zip(targets).map do |p, t|
+        error = t - p
+        error > 0 ? quantile * error : (quantile - 1) * error
+      end.sum
+      sum / n.to_f
+    end
+  end
+end

data/lib/grydra/metrics.rb

@@ -0,0 +1,75 @@
+module GRYDRA
+  module Metrics
+    # Mean Squared Error
+    def self.mse(predictions, actuals)
+      n = predictions.size
+      sum = predictions.zip(actuals).map { |p, r| (p - r)**2 }.sum
+      sum / n.to_f
+    end
+
+    # Mean Absolute Error
+    def self.mae(predictions, actuals)
+      n = predictions.size
+      sum = predictions.zip(actuals).map { |p, r| (p - r).abs }.sum
+      sum / n.to_f
+    end
+
+    # Precision
+    def self.precision(tp, fp)
+      return 0.0 if (tp + fp).zero?
+      tp.to_f / (tp + fp)
+    end
+
+    # Recall
+    def self.recall(tp, fn)
+      return 0.0 if (tp + fn).zero?
+      tp.to_f / (tp + fn)
+    end
+
+    # F1 Score
+    def self.f1(precision, recall)
+      return 0.0 if (precision + recall).zero?
+      2 * (precision * recall) / (precision + recall)
+    end
+
+    # Confusion Matrix
+    def self.confusion_matrix(predictions, actuals, threshold = 0.5)
+      tp = fp = tn = fn = 0
+      predictions.zip(actuals).each do |pred, actual|
+        pred_bin = pred > threshold ? 1 : 0
+        case [pred_bin, actual]
+        when [1, 1] then tp += 1
+        when [1, 0] then fp += 1
+        when [0, 0] then tn += 1
+        when [0, 1] then fn += 1
+        end
+      end
+      { tp: tp, fp: fp, tn: tn, fn: fn }
+    end
+
+    # Area Under the ROC Curve
+    def self.auc_roc(predictions, actuals)
+      pairs = predictions.zip(actuals).sort_by { |pred, _| -pred }
+      positives = actuals.count(1)
+      negatives = actuals.count(0)
+      return 0.5 if positives == 0 || negatives == 0
+
+      auc = 0.0
+      fp = 0
+      pairs.each do |_, actual|
+        if actual == 1
+          auc += fp
+        else
+          fp += 1
+        end
+      end
+      auc / (positives * negatives).to_f
+    end
+
+    # Accuracy
+    def self.accuracy(predictions, actuals, threshold = 0.5)
+      correct = predictions.zip(actuals).count { |pred, actual| (pred > threshold ? 1 : 0) == actual }
+      correct.to_f / predictions.size
+    end
+  end
+end

data/lib/grydra/networks/easy_network.rb

@@ -0,0 +1,161 @@
+module GRYDRA
+  module Networks
+    # Easy Network - Simplified interface for quick training
+    class EasyNetwork
+      attr_accessor :network, :vocabulary, :max_values, :max_values_output
+
+      def initialize(print_epochs: false, plot: false, verbose: true)
+        @network = MainNetwork.new(print_epochs: print_epochs, plot: plot)
+        @vocabulary = nil
+        @max_values = {}
+        @max_values_output = {}
+        @verbose = verbose
+      end
+
+      def configure_adam_optimizer(alpha = 0.001, beta1 = 0.9, beta2 = 0.999)
+        @network.subnets.each { |subnet| subnet.use_adam_optimizer(alpha, beta1, beta2) }
+      end
+
+      def evaluate_model(data_test_x, data_test_y, metrics = %i[mse mae], normalization = :max)
+        predictions = predict_numerical(data_test_x, normalization)
+        results = {}
+        
+        # Handle both single and multi-output cases
+        predictions_flat = predictions.flatten
+        actuals_flat = data_test_y.flatten
+        
+        # Ensure we have valid data
+        return results if predictions_flat.empty? || actuals_flat.empty?
+        return results if predictions_flat.any?(&:nil?) || actuals_flat.any?(&:nil?)
+
+        metrics.each do |metric|
+          case metric
+          when :mse
+            results[:mse] = Metrics.mse(predictions_flat, actuals_flat)
+          when :mae
+            results[:mae] = Metrics.mae(predictions_flat, actuals_flat)
+          when :accuracy
+            results[:accuracy] = Metrics.accuracy(predictions_flat, actuals_flat)
+          when :confusion_matrix
+            results[:confusion_matrix] = Metrics.confusion_matrix(predictions_flat, actuals_flat)
+          when :auc_roc
+            results[:auc_roc] = Metrics.auc_roc(predictions_flat, actuals_flat)
+          end
+        end
+        results
+      end
+
+      # Train with hash data
+      def train_hashes(data_hash, input_keys, label_key, structures, rate, epochs, normalization = :max, **opts)
+        @network.subnets.clear
+
+        inputs = data_hash.map do |item|
+          input_keys.map do |key|
+            value = item[key]
+            value == true ? 1.0 : (value == false ? 0.0 : value.to_f)
+          end
+        end
+
+        @max_values = Normalization.calculate_max_values(inputs, normalization)
+        data_normalized = Normalization.normalize_multiple(inputs, @max_values, normalization)
+
+        labels = data_hash.map { |item| [item[label_key].to_f] }
+        @max_values_output = Normalization.calculate_max_values(labels, normalization)
+        labels_no = Normalization.normalize_multiple(labels, @max_values_output, normalization)
+
+        structures.each do |structure|
+          @network.add_subnet([input_keys.size, *structure])
+        end
+
+        data_for_subnets = structures.map { |_| { input: data_normalized, output: labels_no } }
+        @network.train_subnets(data_for_subnets, rate, epochs, **opts)
+      end
+
+      def predict_hashes(new_hashes, input_keys, normalization = :max)
+        inputs = new_hashes.map do |item|
+          input_keys.map do |key|
+            value = item[key]
+            value == true ? 1.0 : (value == false ? 0.0 : value.to_f)
+          end
+        end
+
+        data_normalized = Normalization.normalize_multiple(inputs, @max_values, normalization)
+        denormalize_predictions(data_normalized, normalization)
+      end
+
+      # Train with numerical data
+      def train_numerical(data_input, data_output, structures, rate, epochs, normalization = :max, **opts)
+        @network.subnets.clear
+
+        @max_values = Normalization.calculate_max_values(data_input, normalization)
+        @max_values_output = Normalization.calculate_max_values(data_output, normalization)
+
+        data_input_no = Normalization.normalize_multiple(data_input, @max_values, normalization)
+        data_output_no = Normalization.normalize_multiple(data_output, @max_values_output, normalization)
+
+        structures.each do |structure|
+          @network.add_subnet([data_input.first.size, *structure])
+        end
+
+        data_for_subnets = structures.map { |_| { input: data_input_no, output: data_output_no } }
+        @network.train_subnets(data_for_subnets, rate, epochs, **opts)
+      end
+
+      def predict_numerical(new_data, normalization = :max)
+        data_normalized = Normalization.normalize_multiple(new_data, @max_values, normalization)
+        denormalize_predictions(data_normalized, normalization)
+      end
+
+      # Train with text data
+      def train_text(texts, labels, structures, rate, epochs, normalization = :max, **opts)
+        @network.subnets.clear
+        @vocabulary = Preprocessing::Text.create_vocabulary(texts)
+
+        inputs = texts.map { |text| Preprocessing::Text.vectorize_text(text, @vocabulary) }
+        
+        # For text, use simple max normalization
+        max_val = @vocabulary.size.to_f
+        data_normalized = inputs.map { |input| input.map { |v| v / max_val } }
+        @max_values = { 0 => max_val }
+
+        @max_values_output = Normalization.calculate_max_values(labels, normalization)
+        labels_no = Normalization.normalize_multiple(labels, @max_values_output, normalization)
+
+        structures.each do |structure|
+          @network.add_subnet([@vocabulary.size, *structure])
+        end
+
+        data_for_subnets = structures.map { |_| { input: data_normalized, output: labels_no } }
+        @network.train_subnets(data_for_subnets, rate, epochs, **opts)
+      end
+
+      def predict_text(new_texts, normalization = :max)
+        inputs = new_texts.map { |text| Preprocessing::Text.vectorize_text(text, @vocabulary) }
+        
+        # For text, we use a simple max normalization based on vocabulary size
+        max_val = @vocabulary.size.to_f
+        data_normalized = inputs.map { |input| input.map { |v| v / max_val } }
+        
+        denormalize_predictions(data_normalized, normalization)
+      end
+
+      private
+
+      def denormalize_predictions(data_normalized, normalization)
+        data_normalized.map do |input|
+          pred_norm = @network.combine_results(input)
+          if normalization == :zscore && @max_values_output.is_a?(Hash) && @max_values_output.key?(:std_devs)
+            pred_norm.map.with_index do |val, idx|
+              val * @max_values_output[:std_devs][idx] + @max_values_output[:means][idx]
+            end
+          else
+            pred_norm.map.with_index do |val, idx|
+              max_val = @max_values_output[idx] || @max_values_output.values.first || 1.0
+              val * max_val
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/grydra/networks/main_network.rb

@@ -0,0 +1,38 @@
+module GRYDRA
+  module Networks
+    # Main Network with multiple subnets
+    class MainNetwork
+      attr_accessor :subnets
+
+      def initialize(print_epochs: false, plot: false)
+        @subnets = []
+        @print_epochs = print_epochs
+        @plot = plot
+      end
+
+      def add_subnet(structure, activations = nil)
+        @subnets << NeuralNetwork.new(structure, print_epochs: @print_epochs, plot: @plot, activations: activations)
+      end
+
+      def train_subnets(data, learning_rate, epochs, **opts)
+        data.each_with_index do |data_subnet, index|
+          puts "Training Subnet #{index + 1}..."
+          @subnets[index].train(data_subnet[:input], data_subnet[:output], learning_rate, epochs, **opts)
+        end
+      end
+
+      def combine_results(input_main)
+        outputs_subnets = @subnets.map { |subnet| subnet.calculate_outputs(input_main) }
+        outputs_subnets.transpose.map { |outputs| outputs.sum / outputs.size }
+      end
+
+      def combine_results_weighted(input_main, weights = nil)
+        outputs_subnets = @subnets.map { |subnet| subnet.calculate_outputs(input_main) }
+        weights ||= Array.new(@subnets.size, 1.0 / @subnets.size)
+        outputs_subnets.transpose.map do |outputs|
+          outputs.zip(weights).map { |output, weight| output * weight }.sum
+        end
+      end
+    end
+  end
+end

data/lib/grydra/networks/neural_network.rb

@@ -0,0 +1,203 @@
+module GRYDRA
+  module Networks
+    # Neural Network class
+    class NeuralNetwork
+      attr_accessor :layers, :history_error
+
+      def initialize(structure, print_epochs: false, plot: false, activations: nil)
+        @print_epochs = print_epochs
+        @plot = plot
+        @layers = []
+        @history_error = []
+        @optimizer = nil
+
+        activations ||= Array.new(structure.size - 1, :tanh)
+
+        structure.each_cons(2).with_index do |(inputs, outputs), i|
+          @layers << Layers::Dense.new(outputs, inputs, activations[i])
+        end
+      end
+
+      def use_adam_optimizer(alpha = 0.001, beta1 = 0.9, beta2 = 0.999)
+        @optimizer = Optimizers::Adam.new(alpha: alpha, beta1: beta1, beta2: beta2)
+      end
+
+      def calculate_outputs(inputs, apply_dropout = false, dropout_rate = 0.5)
+        unless inputs.is_a?(Array) && inputs.all? { |e| e.is_a?(Numeric) }
+          raise ArgumentError, 'Inputs must be an array of numbers'
+        end
+
+        @layers.inject(inputs) { |outputs, layer| layer.calculate_outputs(outputs, apply_dropout, dropout_rate) }
+      end
+
+      # Training with mini-batch, early stopping, decay learning rate, regularization and validation
+      def train(data_input, data_output, learning_rate, epochs, 
+                error_threshold: nil, batch_size: 1, patience: 50, decay: 0.95, 
+                lambda_l1: 0, lambda_l2: 0, dropout: false, dropout_rate: 0.5, 
+                validation_split: 0.0)
+        best_error = Float::INFINITY
+        patience_counter = 0
+        
+        # Split validation data if requested
+        if validation_split > 0
+          split_idx = (data_input.size * (1 - validation_split)).to_i
+          indices = (0...data_input.size).to_a.shuffle
+          train_indices = indices[0...split_idx]
+          val_indices = indices[split_idx..-1]
+          
+          train_input = train_indices.map { |i| data_input[i] }
+          train_output = train_indices.map { |i| data_output[i] }
+          val_input = val_indices.map { |i| data_input[i] }
+          val_output = val_indices.map { |i| data_output[i] }
+        else
+          train_input = data_input
+          train_output = data_output
+          val_input = nil
+          val_output = nil
+        end
+
+        epochs.times do |epoch|
+          error_total = 0
+          error_regularization = 0
+
+          # Shuffle training data
+          indices = (0...train_input.size).to_a.shuffle
+          train_input = indices.map { |i| train_input[i] }
+          train_output = indices.map { |i| train_output[i] }
+
+          data_input.each_slice(batch_size).with_index do |batch_inputs, batch_idx|
+            batch_outputs_real = data_output[batch_idx * batch_size, batch_size]
+
+            batch_inputs.zip(batch_outputs_real).each do |input, output_real|
+              outputs = calculate_outputs(input, dropout, dropout_rate)
+              errors = outputs.zip(output_real).map { |output, real| real - output }
+              error_total += errors.map { |e| e**2 }.sum / errors.size
+
+              # Calculate regularization penalty
+              if lambda_l1 > 0 || lambda_l2 > 0
+                @layers.each do |layer|
+                  layer.neurons.each do |neuron|
+                    error_regularization += Regularization.l1_regularization(neuron.weights, lambda_l1) if lambda_l1 > 0
+                    error_regularization += Regularization.l2_regularization(neuron.weights, lambda_l2) if lambda_l2 > 0
+                  end
+                end
+              end
+
+              # Backpropagation
+              backpropagate(input, errors, learning_rate, lambda_l1, lambda_l2)
+            end
+          end
+
+          error_total += error_regularization
+
+          # Early stopping check
+          if error_threshold && error_total < error_threshold
+            puts "Error threshold reached at epoch #{epoch + 1}: #{error_total}"
+            break
+          end
+
+          if error_total < best_error
+            best_error = error_total
+            patience_counter = 0
+          else
+            patience_counter += 1
+            if patience_counter >= patience
+              puts "Early stopping at epoch #{epoch + 1}, error has not improved for #{patience} epochs."
+              break
+            end
+          end
+
+          learning_rate *= decay
+          @history_error << error_total if @plot
+
+          if @print_epochs
+            puts "Epoch #{epoch + 1}, Total Error: #{error_total.round(6)}, learning rate: #{learning_rate.round(6)}"
+          end
+        end
+
+        Utils::Visualization.plot_error(@history_error) if @plot
+      end
+
+      def info_network
+        puts "Neural network with #{@layers.size} layers:"
+        @layers.each_with_index do |layer, i|
+          puts " Layer #{i + 1}: #{layer.neurons.size} neurons, activation: #{layer.activation}"
+          layer.neurons.each_with_index do |neuron, j|
+            puts "  Neuron #{j + 1}: Weights=#{neuron.weights.map { |p| p.round(3) }}, Bias=#{neuron.bias.round(3)}"
+          end
+        end
+      end
+
+      # Export to DOT for Graphviz
+      def export_graphviz(filename = 'neural_network.dot')
+        File.open(filename, 'w') do |f|
+          f.puts 'digraph NeuralNetwork {'
+          @layers.each_with_index do |layer, i|
+            layer.neurons.each_with_index do |_neuron, j|
+              node = "L#{i}_N#{j}"
+              f.puts "  #{node} [label=\"N#{j + 1}\"];"
+              next unless i < @layers.size - 1
+
+              @layers[i + 1].neurons.each_with_index do |next_neuron, k|
+                weight = next_neuron.weights[j].round(3)
+                f.puts "  #{node} -> L#{i + 1}_N#{k} [label=\"#{weight}\"];"
+              end
+            end
+          end
+          f.puts '}'
+        end
+        puts "Network exported to #{filename} (Graphviz DOT)"
+      end
+
+      private
+
+      def backpropagate(input, errors, learning_rate, lambda_l1, lambda_l2)
+        # Output layer
+        @layers.last.neurons.each_with_index do |neuron, i|
+          neuron.delta = errors[i] * neuron.derivative_activation
+        end
+
+        # Hidden layers
+        (@layers.size - 2).downto(0) do |i|
+          @layers[i].neurons.each_with_index do |neuron, j|
+            sum_deltas = @layers[i + 1].neurons.sum { |n| n.weights[j] * n.delta }
+            neuron.delta = sum_deltas * neuron.derivative_activation
+          end
+        end
+
+        # Update weights and biases
+        @layers.each_with_index do |layer, idx|
+          inputs_layer = idx.zero? ? input : @layers[idx - 1].neurons.map(&:output)
+          layer.neurons.each_with_index do |neuron, neuron_idx|
+            neuron.weights.each_with_index do |_weight, i|
+              gradient = neuron.delta * inputs_layer[i]
+
+              if @optimizer
+                param_id = "layer_#{idx}_neuron_#{neuron_idx}_weight_#{i}"
+                update = @optimizer.update(param_id, gradient)
+                neuron.weights[i] += update
+              else
+                neuron.weights[i] += learning_rate * gradient
+              end
+
+              # Apply regularization
+              if lambda_l1 > 0
+                neuron.weights[i] -= learning_rate * lambda_l1 * (neuron.weights[i] > 0 ? 1 : -1)
+              end
+              neuron.weights[i] -= learning_rate * lambda_l2 * 2 * neuron.weights[i] if lambda_l2 > 0
+            end
+
+            # Update bias
+            if @optimizer
+              param_id = "layer_#{idx}_neuron_#{neuron_idx}_bias"
+              update = @optimizer.update(param_id, neuron.delta)
+              neuron.bias += update
+            else
+              neuron.bias += learning_rate * neuron.delta
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/grydra/networks/neuron.rb

@@ -0,0 +1,80 @@
+module GRYDRA
+  module Networks
+    # Individual Neuron class
+    class Neuron
+      attr_accessor :weights, :bias, :output, :delta
+
+      def initialize(inputs, activation = :tanh)
+        unless inputs.is_a?(Integer) && inputs > 0
+          raise ArgumentError, 'Number of inputs must be a positive integer'
+        end
+
+        @weights = case activation
+                   when :relu, :leaky_relu
+                     Array.new(inputs) { Initializers.he_init(inputs) }
+                   else
+                     Array.new(inputs) { Initializers.xavier_init(inputs) }
+                   end
+
+        @bias = case activation
+                when :relu, :leaky_relu
+                  Initializers.he_init(inputs)
+                else
+                  Initializers.xavier_init(inputs)
+                end
+
+        @output = 0
+        @delta = 0
+        @activation = activation
+        @sum = 0
+        @dropout_mask = nil
+      end
+
+      def calculate_output(inputs, apply_dropout = false, dropout_rate = 0.5)
+        unless inputs.is_a?(Array) && inputs.all? { |e| e.is_a?(Numeric) }
+          raise ArgumentError, 'Inputs must be an array of numbers'
+        end
+
+        if @weights.size != inputs.size
+          raise ArgumentError, "Error: inputs (#{inputs.size}) do not match weights (#{@weights.size})"
+        end
+
+        @sum = @weights.zip(inputs).map { |weight, input| weight * input }.sum + @bias
+        @output = apply_activation(@sum)
+
+        if apply_dropout
+          @dropout_mask = rand < dropout_rate ? 0 : 1 / (1 - dropout_rate)
+          @output *= @dropout_mask
+        end
+
+        @output
+      end
+
+      def derivative_activation
+        case @activation
+        when :tanh then Activations.derivative_tanh(@output)
+        when :relu then Activations.derivative_relu(@output)
+        when :sigmoid then Activations.derivative_sigmoid(@sum)
+        when :leaky_relu then Activations.derivative_leaky_relu(@sum)
+        when :swish then Activations.derivative_swish(@sum)
+        when :gelu then Activations.derivative_gelu(@sum)
+        else 1
+        end
+      end
+
+      private
+
+      def apply_activation(value)
+        case @activation
+        when :tanh then Activations.tanh(value)
+        when :relu then Activations.relu(value)
+        when :sigmoid then Activations.sigmoid(value)
+        when :leaky_relu then Activations.leaky_relu(value)
+        when :swish then Activations.swish(value)
+        when :gelu then Activations.gelu(value)
+        else value
+        end
+      end
+    end
+  end
+end