grydra 1.0.0 → 2.0.1

data/lib/grydra/activations.rb

@@ -0,0 +1,70 @@
+module GRYDRA
+  module Activations
+    # Hyperbolic tangent activation function
+    def self.tanh(x)
+      Math.tanh(x)
+    end
+
+    def self.derivative_tanh(x)
+      1 - tanh(x)**2
+    end
+
+    # Rectified Linear Unit
+    def self.relu(x)
+      x > 0 ? x : 0
+    end
+
+    def self.derivative_relu(x)
+      x > 0 ? 1 : 0
+    end
+
+    # Sigmoid activation function
+    def self.sigmoid(x)
+      1.0 / (1.0 + Math.exp(-x))
+    end
+
+    def self.derivative_sigmoid(x)
+      s = sigmoid(x)
+      s * (1 - s)
+    end
+
+    # Softmax for multi-class classification
+    def self.softmax(vector)
+      max = vector.max
+      exps = vector.map { |v| Math.exp(v - max) }
+      sum = exps.sum
+      exps.map { |v| v / sum }
+    end
+
+    # Leaky ReLU
+    def self.leaky_relu(x, alpha = 0.01)
+      x > 0 ? x : alpha * x
+    end
+
+    def self.derivative_leaky_relu(x, alpha = 0.01)
+      x > 0 ? 1 : alpha
+    end
+
+    # Swish activation function
+    def self.swish(x)
+      x * sigmoid(x)
+    end
+
+    def self.derivative_swish(x)
+      s = sigmoid(x)
+      s + x * s * (1 - s)
+    end
+
+    # GELU (Gaussian Error Linear Unit)
+    def self.gelu(x)
+      0.5 * x * (1 + Math.tanh(Math.sqrt(2 / Math::PI) * (x + 0.044715 * x**3)))
+    end
+
+    def self.derivative_gelu(x)
+      tanh_arg = Math.sqrt(2 / Math::PI) * (x + 0.044715 * x**3)
+      tanh_val = Math.tanh(tanh_arg)
+      sech2 = 1 - tanh_val**2
+      0.5 * (1 + tanh_val) + 0.5 * x * sech2 * Math.sqrt(2 / Math::PI) * (1 + 3 * 0.044715 * x**2)
+    end
+  end
+end

data/lib/grydra/callbacks.rb

@@ -0,0 +1,212 @@
+module GRYDRA
+  module Callbacks
+    # Base Callback class
+    class Base
+      def on_train_begin(logs = {}); end
+      def on_train_end(logs = {}); end
+      def on_epoch_begin(epoch, logs = {}); end
+      def on_epoch_end(epoch, logs = {}); end
+      def on_batch_begin(batch, logs = {}); end
+      def on_batch_end(batch, logs = {}); end
+    end
+
+    # Early Stopping Callback
+    class EarlyStopping < Base
+      attr_reader :stopped_epoch
+
+      def initialize(monitor: :loss, patience: 10, min_delta: 0.0, mode: :min, restore_best: true)
+        @monitor = monitor
+        @patience = patience
+        @min_delta = min_delta
+        @mode = mode
+        @restore_best = restore_best
+        @wait = 0
+        @stopped_epoch = 0
+        @best_value = mode == :min ? Float::INFINITY : -Float::INFINITY
+        @best_weights = nil
+      end
+
+      def on_epoch_end(epoch, logs = {})
+        current = logs[@monitor]
+        return unless current
+
+        if improved?(current)
+          @best_value = current
+          @wait = 0
+          @best_weights = logs[:weights].dup if @restore_best && logs[:weights]
+        else
+          @wait += 1
+          if @wait >= @patience
+            @stopped_epoch = epoch
+            logs[:stop_training] = true
+            logs[:weights] = @best_weights if @restore_best && @best_weights
+            puts "Early stopping at epoch #{epoch + 1}"
+          end
+        end
+      end
+
+      private
+
+      def improved?(current)
+        if @mode == :min
+          current < @best_value - @min_delta
+        else
+          current > @best_value + @min_delta
+        end
+      end
+    end
+
+    # Learning Rate Scheduler
+    class LearningRateScheduler < Base
+      def initialize(schedule: nil, &block)
+        @schedule = schedule || block
+      end
+
+      def on_epoch_begin(epoch, logs = {})
+        new_lr = @schedule.call(epoch, logs[:learning_rate])
+        logs[:learning_rate] = new_lr if new_lr
+      end
+    end
+
+    # Reduce Learning Rate on Plateau
+    class ReduceLROnPlateau < Base
+      def initialize(monitor: :loss, factor: 0.5, patience: 5, min_lr: 1e-7, mode: :min)
+        @monitor = monitor
+        @factor = factor
+        @patience = patience
+        @min_lr = min_lr
+        @mode = mode
+        @wait = 0
+        @best_value = mode == :min ? Float::INFINITY : -Float::INFINITY
+      end
+
+      def on_epoch_end(epoch, logs = {})
+        current = logs[@monitor]
+        return unless current
+
+        if improved?(current)
+          @best_value = current
+          @wait = 0
+        else
+          @wait += 1
+          if @wait >= @patience
+            old_lr = logs[:learning_rate]
+            new_lr = [old_lr * @factor, @min_lr].max
+            if new_lr < old_lr
+              logs[:learning_rate] = new_lr
+              puts "Reducing learning rate to #{new_lr.round(8)}"
+            end
+            @wait = 0
+          end
+        end
+      end
+
+      private
+
+      def improved?(current)
+        if @mode == :min
+          current < @best_value
+        else
+          current > @best_value
+        end
+      end
+    end
+
+    # Model Checkpoint Callback
+    class ModelCheckpoint < Base
+      def initialize(filepath, monitor: :loss, save_best_only: true, mode: :min)
+        @filepath = filepath
+        @monitor = monitor
+        @save_best_only = save_best_only
+        @mode = mode
+        @best_value = mode == :min ? Float::INFINITY : -Float::INFINITY
+      end
+
+      def on_epoch_end(epoch, logs = {})
+        current = logs[@monitor]
+        
+        if @save_best_only
+          if improved?(current)
+            @best_value = current
+            save_model(logs[:model], epoch)
+          end
+        else
+          save_model(logs[:model], epoch)
+        end
+      end
+
+      private
+
+      def improved?(current)
+        if @mode == :min
+          current < @best_value
+        else
+          current > @best_value
+        end
+      end
+
+      def save_model(model, epoch)
+        filepath = @filepath.gsub('{epoch}', epoch.to_s)
+        Utils::Persistence.save_model(model, filepath)
+      end
+    end
+
+    # CSV Logger Callback
+    class CSVLogger < Base
+      def initialize(filename, separator: ',', append: false)
+        @filename = filename
+        @separator = separator
+        @append = append
+        @file = nil
+        @keys = nil
+      end
+
+      def on_train_begin(logs = {})
+        mode = @append ? 'a' : 'w'
+        @file = File.open(@filename, mode)
+      end
+
+      def on_epoch_end(epoch, logs = {})
+        if @keys.nil?
+          @keys = ['epoch'] + logs.keys.map(&:to_s).sort
+          @file.puts @keys.join(@separator) unless @append
+        end
+
+        values = [epoch] + @keys[1..-1].map { |k| logs[k.to_sym] || '' }
+        @file.puts values.join(@separator)
+        @file.flush
+      end
+
+      def on_train_end(logs = {})
+        @file.close if @file
+      end
+    end
+
+    # Progress Bar Callback
+    class ProgressBar < Base
+      def initialize(total_epochs)
+        @total_epochs = total_epochs
+        @start_time = nil
+      end
+
+      def on_train_begin(logs = {})
+        @start_time = Time.now
+      end
+
+      def on_epoch_end(epoch, logs = {})
+        progress = (epoch + 1).to_f / @total_epochs
+        bar_length = 30
+        filled = (bar_length * progress).to_i
+        bar = '=' * filled + '-' * (bar_length - filled)
+        
+        elapsed = Time.now - @start_time
+        eta = elapsed / progress - elapsed
+        
+        metrics = logs.map { |k, v| "#{k}: #{v.is_a?(Numeric) ? v.round(6) : v}" }.join(', ')
+        
+        print "\rEpoch #{epoch + 1}/#{@total_epochs} [#{bar}] - #{elapsed.round(1)}s - ETA: #{eta.round(1)}s - #{metrics}"
+        puts if epoch + 1 == @total_epochs
+      end
+    end
+  end
+end

data/lib/grydra/documentation.rb

@@ -0,0 +1,84 @@
+module GRYDRA
+  module Documentation
+    METHOD_DESCRIPTIONS = {
+      # Networks
+      'Networks::MainNetwork.add_subnet' => {
+        description: 'Adds a subnet to the main network with the given structure.',
+        example: 'network = GRYDRA::Networks::MainNetwork.new; network.add_subnet([2, 4, 1])'
+      },
+      'Networks::EasyNetwork.train_numerical' => {
+        description: 'Trains the network with numerical data.',
+        example: 'network.train_numerical(data_input, data_output, [[4, 1]], 0.05, 15000, :max)'
+      },
+      'Networks::EasyNetwork.configure_adam_optimizer' => {
+        description: 'Configures the Adam optimizer for all subnets.',
+        example: 'network.configure_adam_optimizer(0.001, 0.9, 0.999)'
+      },
+      
+      # Metrics
+      'Metrics.mse' => {
+        description: 'Calculates Mean Squared Error.',
+        example: 'GRYDRA::Metrics.mse(predictions, actuals)'
+      },
+      'Metrics.accuracy' => {
+        description: 'Calculates classification accuracy.',
+        example: 'GRYDRA::Metrics.accuracy(predictions, actuals, 0.5)'
+      },
+      
+      # Persistence
+      'Utils::Persistence.save_model' => {
+        description: 'Saves the trained model to a binary file.',
+        example: 'GRYDRA::Utils::Persistence.save_model(model, "my_model", "./models")'
+      },
+      'Utils::Persistence.load_model' => {
+        description: 'Loads a saved model from a binary file.',
+        example: 'model = GRYDRA::Utils::Persistence.load_model("my_model", "./models")'
+      },
+      
+      # Visualization
+      'Utils::Visualization.plot_architecture_ascii' => {
+        description: 'Displays an ASCII representation of the network architecture.',
+        example: 'GRYDRA::Utils::Visualization.plot_architecture_ascii(model)'
+      },
+      'Utils::Visualization.analyze_gradients' => {
+        description: 'Analyzes the model\'s gradients.',
+        example: 'analysis = GRYDRA::Utils::Visualization.analyze_gradients(model)'
+      },
+      
+      # Training
+      'Training::CrossValidation.cross_validation' => {
+        description: 'Performs k-fold cross-validation.',
+        example: 'result = GRYDRA::Training::CrossValidation.cross_validation(data_x, data_y, 5) { |train_x, train_y, test_x, test_y| ... }'
+      },
+      'Training::HyperparameterSearch.hyperparameter_search' => {
+        description: 'Performs hyperparameter search using grid search.',
+        example: 'result = GRYDRA::Training::HyperparameterSearch.hyperparameter_search(data_x, data_y, param_grid) { |params, x, y| ... }'
+      }
+    }
+
+    def self.describe_method(class_name, method_name)
+      key = "#{class_name}.#{method_name}"
+      info = METHOD_DESCRIPTIONS[key]
+
+      if info
+        puts "\e[1;36m📖 Description of #{key}:\e[0m"
+        puts info[:description]
+        puts "\n💡 Example:"
+        puts "#{info[:example]}\e[0m"
+      else
+        puts "\e[31;1m✗ No description found for method '#{key}'\e[0m"
+        puts "\e[36m💡 You can call: GRYDRA::Documentation.list_methods_available\e[0m"
+      end
+    end
+
+    def self.list_methods_available
+      puts "\e[1;36m📚 Documented public methods:\e[0m"
+      grouped = METHOD_DESCRIPTIONS.keys.group_by { |k| k.split('.').first }
+      grouped.each do |class_name, methods|
+        puts "  #{class_name}:"
+        methods.each { |m| puts "    - #{m.split('.').last}" }
+      end
+      print "\e[0m"
+    end
+  end
+end

data/lib/grydra/initializers.rb

@@ -0,0 +1,14 @@
+module GRYDRA
+  module Initializers
+    # Xavier/Glorot initialization
+    def self.xavier_init(num_inputs)
+      limit = Math.sqrt(6.0 / num_inputs)
+      rand * 2 * limit - limit
+    end
+
+    # He initialization (for ReLU)
+    def self.he_init(num_inputs)
+      Math.sqrt(2.0 / num_inputs) * (rand * 2 - 1)
+    end
+  end
+end

data/lib/grydra/layers/base.rb

@@ -0,0 +1,10 @@
+module GRYDRA
+  module Layers
+    # Base Layer class
+    class Base
+      def calculate_outputs(inputs)
+        raise NotImplementedError, 'Subclasses must implement calculate_outputs method'
+      end
+    end
+  end
+end

data/lib/grydra/layers/conv.rb

@@ -0,0 +1,106 @@
+module GRYDRA
+  module Layers
+    # Convolutional Layer - Full 2D implementation
+    class Conv < Base
+      attr_accessor :filters, :kernel_size, :stride, :padding, :weights, :biases
+
+      def initialize(filters, kernel_size, stride: 1, padding: 0, activation: :relu, input_channels: 1)
+        @filters = filters
+        @kernel_size = kernel_size
+        @stride = stride
+        @padding = padding
+        @activation = activation
+        @input_channels = input_channels
+        
+        # Initialize weights using He initialization for ReLU
+        @weights = Array.new(filters) do
+          Array.new(input_channels) do
+            Array.new(kernel_size) do
+              Array.new(kernel_size) { Initializers.he_init(kernel_size * kernel_size) }
+            end
+          end
+        end
+        @biases = Array.new(filters, 0.0)
+        @output_cache = nil
+      end
+
+      def calculate_outputs(input)
+        # Input shape: [channels, height, width]
+        channels, height, width = input_shape(input)
+        
+        # Apply padding if needed
+        padded_input = apply_padding(input) if @padding > 0
+        padded_input ||= input
+        
+        # Calculate output dimensions
+        out_height = ((height + 2 * @padding - @kernel_size) / @stride) + 1
+        out_width = ((width + 2 * @padding - @kernel_size) / @stride) + 1
+        
+        # Perform convolution
+        output = Array.new(@filters) do |f|
+          Array.new(out_height) do |i|
+            Array.new(out_width) do |j|
+              convolve_at_position(padded_input, f, i, j)
+            end
+          end
+        end
+        
+        @output_cache = output
+        output
+      end
+
+      private
+
+      def input_shape(input)
+        if input.is_a?(Array) && input.first.is_a?(Array) && input.first.first.is_a?(Array)
+          [input.size, input.first.size, input.first.first.size]
+        else
+          # Flatten input assumed
+          [1, 1, input.size]
+        end
+      end
+
+      def apply_padding(input)
+        channels, height, width = input_shape(input)
+        padded = Array.new(channels) do |c|
+          Array.new(height + 2 * @padding) do |i|
+            Array.new(width + 2 * @padding) do |j|
+              if i < @padding || i >= height + @padding || j < @padding || j >= width + @padding
+                0.0
+              else
+                input[c][i - @padding][j - @padding]
+              end
+            end
+          end
+        end
+        padded
+      end
+
+      def convolve_at_position(input, filter_idx, out_i, out_j)
+        sum = @biases[filter_idx]
+        
+        @input_channels.times do |c|
+          @kernel_size.times do |ki|
+            @kernel_size.times do |kj|
+              i = out_i * @stride + ki
+              j = out_j * @stride + kj
+              sum += input[c][i][j] * @weights[filter_idx][c][ki][kj]
+            end
+          end
+        end
+        
+        apply_activation(sum)
+      end
+
+      def apply_activation(value)
+        case @activation
+        when :relu then Activations.relu(value)
+        when :tanh then Activations.tanh(value)
+        when :sigmoid then Activations.sigmoid(value)
+        when :leaky_relu then Activations.leaky_relu(value)
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/grydra/layers/dense.rb

@@ -0,0 +1,17 @@
+module GRYDRA
+  module Layers
+    # Dense (Fully Connected) Layer
+    class Dense < Base
+      attr_accessor :neurons, :activation
+
+      def initialize(num_neurons, inputs_per_neuron, activation = :tanh)
+        @activation = activation
+        @neurons = Array.new(num_neurons) { Networks::Neuron.new(inputs_per_neuron, activation) }
+      end
+
+      def calculate_outputs(inputs, apply_dropout = false, dropout_rate = 0.5)
+        @neurons.map { |neuron| neuron.calculate_output(inputs, apply_dropout, dropout_rate) }
+      end
+    end
+  end
+end

data/lib/grydra/layers/lstm.rb

@@ -0,0 +1,139 @@
+module GRYDRA
+  module Layers
+    # LSTM Layer - Full implementation with backpropagation support
+    class LSTM < Base
+      attr_accessor :units, :cell_state, :hidden_state
+      attr_reader :weights, :biases
+
+      def initialize(units, inputs_per_unit, return_sequences: false)
+        @units = units
+        @inputs_per_unit = inputs_per_unit
+        @return_sequences = return_sequences
+        
+        combined_size = inputs_per_unit + units
+        
+        # Initialize weight matrices for all gates
+        @weights = {
+          forget: initialize_weights(combined_size, units),
+          input: initialize_weights(combined_size, units),
+          candidate: initialize_weights(combined_size, units),
+          output: initialize_weights(combined_size, units)
+        }
+        
+        @biases = {
+          forget: Array.new(units, 1.0),  # Bias forget gate to 1 initially
+          input: Array.new(units, 0.0),
+          candidate: Array.new(units, 0.0),
+          output: Array.new(units, 0.0)
+        }
+        
+        # Cache for backpropagation
+        @cache = []
+        
+        reset_state
+      end
+
+      def reset_state
+        @cell_state = Array.new(@units, 0.0)
+        @hidden_state = Array.new(@units, 0.0)
+        @cache.clear
+      end
+
+      def calculate_outputs(input_sequence)
+        outputs = []
+        
+        input_sequence = [input_sequence] unless input_sequence.first.is_a?(Array)
+        
+        input_sequence.each do |input|
+          combined = input + @hidden_state
+          
+          # Forget gate
+          f_t = gate_activation(combined, @weights[:forget], @biases[:forget], :sigmoid)
+          
+          # Input gate
+          i_t = gate_activation(combined, @weights[:input], @biases[:input], :sigmoid)
+          
+          # Candidate values
+          c_tilde = gate_activation(combined, @weights[:candidate], @biases[:candidate], :tanh)
+          
+          # Update cell state
+          @cell_state = @cell_state.zip(f_t, i_t, c_tilde).map do |c, f, i, c_t|
+            f * c + i * c_t
+          end
+          
+          # Output gate
+          o_t = gate_activation(combined, @weights[:output], @biases[:output], :sigmoid)
+          
+          # Update hidden state
+          @hidden_state = o_t.zip(@cell_state).map { |o, c| o * Math.tanh(c) }
+          
+          # Cache for backpropagation
+          @cache << {
+            input: input,
+            combined: combined,
+            f_t: f_t,
+            i_t: i_t,
+            c_tilde: c_tilde,
+            o_t: o_t,
+            cell_state: @cell_state.dup,
+            hidden_state: @hidden_state.dup
+          }
+          
+          outputs << @hidden_state.dup
+        end
+        
+        @return_sequences ? outputs : outputs.last
+      end
+
+      def backward(d_hidden, learning_rate: 0.01)
+        # Simplified backpropagation through time
+        d_cell = Array.new(@units, 0.0)
+        
+        @cache.reverse.each do |cache|
+          # Gradient through output gate
+          d_o = d_hidden.zip(cache[:cell_state]).map { |dh, c| dh * Math.tanh(c) }
+          d_cell = d_hidden.zip(cache[:o_t], cache[:cell_state]).map do |dh, o, c|
+            d_cell_val = dh * o * (1 - Math.tanh(c)**2)
+            d_cell_val
+          end
+          
+          # Update weights (simplified)
+          update_gate_weights(:output, cache[:combined], d_o, learning_rate)
+        end
+      end
+
+      private
+
+      def initialize_weights(input_size, output_size)
+        Array.new(output_size) do
+          Array.new(input_size) { Initializers.xavier_init(input_size) }
+        end
+      end
+
+      def gate_activation(input, weights, biases, activation)
+        output = Array.new(@units) do |i|
+          sum = biases[i]
+          input.each_with_index { |x, j| sum += x * weights[i][j] }
+          sum
+        end
+        
+        output.map do |val|
+          case activation
+          when :sigmoid then Activations.sigmoid(val)
+          when :tanh then Activations.tanh(val)
+          else val
+          end
+        end
+      end
+
+      def update_gate_weights(gate, input, gradient, learning_rate)
+        gradient.each_with_index do |grad, i|
+          input.each_with_index do |inp, j|
+            @weights[gate][i][j] += learning_rate * grad * inp
+          end
+          @biases[gate][i] += learning_rate * grad
+        end
+      end
+    end
+  end
+end