qoa 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a45d057fc9994389ef429aa82b3207a2e30b8e67a880f3ad6e6b57f7e7a79b75
-  data.tar.gz: 273f73d4184545b75ec338f75c9ea39b5e4e2cc6e2582091bab036e93f06b320
+  metadata.gz: 8541feace7b5d8df6418672899fbba8289cc63624d7876c640a28e3143a450f3
+  data.tar.gz: 3944346b90a422a1ea843aef9b0a5bd18c8067e0bc075fa897daab0a1cf9373c
 SHA512:
-  metadata.gz: 00eedcfdee31e73c559be6b8c3e75d31c72d2f802a40e9322c82b0bfe6e615ce9f2498d559722fa360d9ebb7d23328f8275c94374b40651d82e3705f869e413a
-  data.tar.gz: c125116a1e5deb7d345315eaa9954e006b198a881864b123b452906dcd1d009f473bbfc82a7cd7f09a67dca7c61de83aefb8156eed4cfde8b5d593858448c017
+  metadata.gz: 8e0afee3076f2d47ff091c2d943c53710e080b015f2773733aabaf90fc4b2ccb26507d24344d1e3b6915e488df875b093577098572c889a1a621064600467037
+  data.tar.gz: aaf6f8e7e9280630fd6218d72a6dd47a4e68602b193a127be8b59462e3204267a28e490e61e8dc19523b8aa463f2d9611fd81617b9da9f79d0e83b1c1942a638

data/README.md

@@ -10,6 +10,7 @@ Qoa is a simple and customizable neural network library for Ruby. It allows you
 - Weight initialization using Xavier initialization
 - Customizable learning parameters
 - Parallelized backward pass for faster training
+- Supports convolutional and pooling layers
 
 ## Installation
 
@@ -34,7 +35,7 @@ require 'qoa'
 To create a new neural network, you can initialize an instance of `Qoa::NeuralNetwork` with the following parameters:
 
 - `input_nodes`: The number of input nodes.
-- `hidden_layers`: An array of the number of nodes in each hidden layer.
+- `hidden_layers`: An array of the number of nodes in each hidden layer, or an array of `[:conv, nodes, kernel_size, stride]` for a convolutional layer or `[:pool, nodes, pool_size, stride]` for a pooling layer.
 - `output_nodes`: The number of output nodes.
 - `learning_rate`: The learning rate for the gradient descent optimization.
 - `dropout_rate`: The dropout rate for regularization.
@@ -42,6 +43,8 @@ To create a new neural network, you can initialize an instance of `Qoa::NeuralNe
 - `decay_rate`: The decay rate for the RMSProp optimizer (default is `0.9`).
 - `epsilon`: A small value to prevent division by zero in the RMSProp optimizer (default is `1e-8`).
 - `batch_size`: The size of the mini-batches used for training (default is `10`).
+- `l1_lambda`: The L1 regularization parameter (default is `0.0`).
+- `l2_lambda`: The L2 regularization parameter (default is `0.0`).
 
 Example:
 
@@ -49,7 +52,7 @@ Example:
 require 'qoa'
 
 input_nodes = 784 # Number of input features (e.g., 28x28 pixels for MNIST dataset)
-hidden_layers = [128, 64] # Two hidden layers with 128 and 64 nodes each
+hidden_layers = [128, [:conv, 64, 3, 1]] # One hidden layer with 128 nodes and one convolutional layer with 64 nodes, kernel size 3, and stride 1
 output_nodes = 10 # Number of output classes (e.g., 10 for MNIST dataset)
 learning_rate = 0.01
 dropout_rate = 0.5

data/lib/qoa/err/validations.rb

@@ -0,0 +1,27 @@
+module Qoa
+  module Err
+    module Validations
+      def validate_constructor_args(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func, decay_rate, epsilon, batch_size, l1_lambda, l2_lambda)
+        raise ArgumentError, 'input_nodes, hidden_layers, and output_nodes must be positive integers' unless [input_nodes, output_nodes].all? { |x| x.is_a?(Integer) && x > 0 } && hidden_layers.is_a?(Array) && hidden_layers.all? { |x| x.is_a?(Integer) && x > 0 }
+        raise ArgumentError, 'learning_rate, dropout_rate, decay_rate, epsilon, l1_lambda, and l2_lambda must be positive numbers' unless [learning_rate, dropout_rate, decay_rate, epsilon, l1_lambda, l2_lambda].all? { |x| x.is_a?(Numeric) && x >= 0 }
+        raise ArgumentError, 'activation_func must be a valid symbol' unless ActivationFunctions.methods.include?(activation_func)
+        raise ArgumentError, 'batch_size must be a positive integer' unless batch_size.is_a?(Integer) && batch_size > 0
+      end
+
+      def validate_query_args(inputs)
+        raise ArgumentError, 'inputs must be an array of numbers' unless inputs.is_a?(Array) && inputs.all? { |x| x.is_a?(Numeric) }
+      end
+
+      def validate_calculate_loss_args(inputs, targets, loss_function)
+        raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
+        raise ArgumentError, 'inputs and targets must be arrays of arrays of numbers' unless inputs.is_a?(Array) && targets.is_a?(Array) && inputs.all? { |x| x.is_a?(Array) && x.all? { |y| y.is_a?(Numeric) } } && targets.all? { |x| x.is_a?(Array) && x.all? { |y| y.is_a?(Numeric) } }
+        raise ArgumentError, 'loss_function must be a valid symbol' unless LossFunctions.methods.include?(loss_function)
+      end
+
+      def validate_train_args(inputs, targets)
+        raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
+        raise ArgumentError, 'inputs and targets must be arrays of arrays of numbers' unless inputs.is_a?(Array) && targets.is_a?(Array) && inputs.all? { |x| x.is_a?(Array) && x.all? { |y| y.is_a?(Numeric) } } && targets.all? { |x| x.is_a?(Array) && x.all? { |y| y.is_a?(Numeric) } }
+      end
+    end
+  end
+end

data/lib/qoa/layers/convolutional_layer.rb

@@ -0,0 +1,18 @@
+module Qoa
+  module Layers
+    class ConvolutionalLayer < Qoa::Layers::Layer
+      attr_reader :kernel_size, :stride
+
+      def initialize(input_size, output_size, kernel_size, stride = 1)
+        super(input_size, output_size)
+        @kernel_size = kernel_size
+        @stride = stride
+      end
+
+      def random_matrix(rows, cols)
+        limit = Math.sqrt(6.0 / (rows + cols))
+        Array.new(rows) { Array.new(cols) { rand(-limit..limit) } }
+      end
+    end
+  end
+end

data/lib/qoa/layers/layer.rb

@@ -0,0 +1,22 @@
+module Qoa
+  module Layers
+    class Layer
+      attr_reader :input_size, :output_size, :weights
+
+      def initialize(input_size, output_size)
+        @input_size = input_size
+        @output_size = output_size
+        @weights = random_matrix(output_size, input_size)
+      end
+
+      def random_matrix(rows, cols)
+        limit = Math.sqrt(6.0 / (rows + cols))
+        Array.new(rows) { Array.new(cols) { rand(-limit..limit) } }
+      end
+
+      def weights=(new_weights)
+        @weights = new_weights
+      end
+    end
+  end
+end

data/lib/qoa/layers/pooling_layer.rb

@@ -0,0 +1,13 @@
+module Qoa
+  module Layers
+    class PoolingLayer < Qoa::Layers::Layer
+      attr_reader :pool_size, :stride
+
+      def initialize(input_size, output_size, pool_size, stride = 1)
+        super(input_size, output_size)
+        @pool_size = pool_size
+        @stride = stride
+      end
+    end
+  end
+end

data/lib/qoa/loss_functions.rb

@@ -0,0 +1,30 @@
+module Qoa
+  module LossFunctions
+    class << self
+      def mean_squared_error(prediction, target)
+        raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+        prediction.zip(target).map { |p, t| (p - t) ** 2 }.sum / prediction.size
+      end
+
+      def cross_entropy_loss(prediction, target)
+        raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+        -prediction.zip(target).map { |p, t| t * Math.log(p) }.sum / prediction.size
+      end
+
+      def binary_cross_entropy(prediction, target)
+        raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+        -prediction.zip(target).map { |p, t| t * Math.log(p) + (1 - t) * Math.log(1 - p) }.sum / prediction.size
+      end
+
+      def categorical_cross_entropy(prediction, target)
+        raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+        -prediction.zip(target).map { |p, t| t * Math.log(p) }.sum / prediction.size
+      end
+
+      def mean_absolute_error(prediction, target)
+        raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+        prediction.zip(target).map { |p, t| (p - t).abs }.sum / prediction.size
+      end
+    end
+  end
+end

data/lib/qoa/matrix_helpers.rb

@@ -24,7 +24,11 @@ module Qoa
     end
 
     def apply_function(matrix, func)
-      matrix.map { |row| row.map { |x| func.call(x) } }
+      matrix.map do |row|
+        row.map do |x|
+          x.nil? ? nil : func.call(x)  # Add a check for nil values
+        end
+      end
     end
 
     def transpose(matrix)

data/lib/qoa/neural_network.rb

@@ -1,15 +1,24 @@
-require_relative 'layer'
+require_relative 'layers/layer'
+require_relative 'layers/convolutional_layer'
+require_relative 'layers/pooling_layer'
 require_relative 'activation_functions'
 require_relative 'training'
 require_relative 'utils'
+require_relative 'loss_functions'
+require_relative 'err/validations'
 
 module Qoa
   class NeuralNetwork
     include Training
     include Utils
-    attr_reader :input_nodes, :hidden_layers, :output_nodes, :learning_rate, :activation_func, :dropout_rate, :decay_rate, :epsilon, :batch_size
+    include LossFunctions
+    include Err::Validations
+
+    attr_reader :input_nodes, :hidden_layers, :output_nodes, :learning_rate, :activation_func, :dropout_rate, :decay_rate, :epsilon, :batch_size, :l1_lambda, :l2_lambda
+
+    def initialize(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func = :leaky_relu, decay_rate = 0.9, epsilon = 1e-8, batch_size = 10, l1_lambda = 0.0, l2_lambda = 0.0)
+      # validate_constructor_args(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func, decay_rate, epsilon, batch_size, l1_lambda, l2_lambda)
 
-    def initialize(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func = :sigmoid, decay_rate = 0.9, epsilon = 1e-8, batch_size = 10)
       @input_nodes = input_nodes
       @hidden_layers = hidden_layers
       @output_nodes = output_nodes
@@ -19,27 +28,47 @@ module Qoa
       @decay_rate = decay_rate
       @epsilon = epsilon
       @batch_size = batch_size
+      @l1_lambda = l1_lambda
+      @l2_lambda = l2_lambda
 
       @layers = []
-      @layers << Layer.new(input_nodes, hidden_layers[0])
+      @layers << Qoa::Layers::Layer.new(input_nodes, hidden_layers[0].is_a?(Array) ? hidden_layers[0][1] : hidden_layers[0])
+
       hidden_layers.each_cons(2) do |l1, l2|
-        @layers << Layer.new(l1, l2)
+        l1_size = l1.is_a?(Array) ? l1[1] : l1
+        l2_size = l2.is_a?(Array) ? l2[1] : l2
+
+        if l1.is_a?(Array) && l2.is_a?(Array) && l1[0] == :conv && l2[0] == :conv
+          @layers << Qoa::Layers::ConvolutionalLayer.new(l1_size, l2_size, l1[2], l1[3])
+        elsif l1.is_a?(Array) && l1[0] == :conv && l2.is_a?(Numeric)
+          @layers << Qoa::Layers::ConvolutionalLayer.new(l1_size, l2_size, l1[2], l1[3])
+        elsif l1.is_a?(Numeric) && l2.is_a?(Array) && l2[0] == :conv
+          @layers << Qoa::Layers::ConvolutionalLayer.new(l1_size, l2_size, l2[2], l2[3])
+        elsif l1.is_a?(Array) && l1[0] == :pool && l2.is_a?(Numeric)
+          @layers << Qoa::Layers::PoolingLayer.new(l1_size, l2_size, l1[2], l1[3])
+        elsif l1.is_a?(Numeric) && l2.is_a?(Array) && l2[0] == :pool
+          @layers << Qoa::Layers::PoolingLayer.new(l1_size, l2_size, l2[2], l2[3])
+        else
+          @layers << Qoa::Layers::Layer.new(l1_size, l2_size)
+        end
       end
-      @layers << Layer.new(hidden_layers[-1], output_nodes)
+      @layers << Qoa::Layers::Layer.new(hidden_layers[-1].is_a?(Array) ? hidden_layers[-1][1] : hidden_layers[-1], output_nodes)
     end
 
     def query(inputs)
+      validate_query_args(inputs)
+
       layer_outputs = forward_pass(inputs)
       layer_outputs.last.flatten
     end
 
-    def calculate_loss(inputs, targets)
-      raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
+    def calculate_loss(inputs, targets, loss_function = :cross_entropy_loss)
+      validate_calculate_loss_args(inputs, targets, loss_function)
 
       total_loss = 0.0
       inputs.zip(targets).each do |input, target|
         prediction = query(input)
-        total_loss += mean_squared_error(prediction, target)
+        total_loss += LossFunctions.send(loss_function, prediction, target)
       end
 
       total_loss / inputs.size

data/lib/qoa/training.rb

@@ -1,12 +1,14 @@
-require_relative 'matrix_helpers'
 require 'concurrent'
+require_relative 'matrix_helpers'
+require_relative 'err/validations'
 
 module Qoa
   module Training
     include MatrixHelpers
+    include Err::Validations
 
     def train(inputs, targets)
-      raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
+      validate_train_args(inputs, targets)
 
       inputs.zip(targets).each_slice(@batch_size) do |batch|
         train_batch(batch)
@@ -44,7 +46,8 @@ module Qoa
 
       # Update weights
       @layers.each_with_index do |layer, i|
-        layer.weights = matrix_add(layer.weights, scalar_multiply(@learning_rate / batch.size, weight_deltas[i]))
+        regularization_penalty = calculate_regularization_penalty(layer.weights, @l1_lambda, @l2_lambda)
+        layer.weights = matrix_add(layer.weights, scalar_multiply(@learning_rate / batch.size, matrix_add(weight_deltas[i], regularization_penalty)))
       end
     end
 
@@ -77,8 +80,15 @@ module Qoa
       inputs = inputs.map { |x| [x] } # Convert to column vector
 
       layer_outputs = [inputs]
-      @layers.map(&:weights).each_with_index do |w, i|
-        layer_inputs = matrix_multiply(w, layer_outputs[-1])
+      @layers.each_with_index do |layer, i|
+        if layer.is_a?(Qoa::Layers::ConvolutionalLayer)
+          layer_inputs = convolution(layer, layer_outputs[-1])
+        elsif layer.is_a?(Qoa::Layers::PoolingLayer)
+          layer_inputs = pooling(layer, layer_outputs[-1])
+        else
+          layer_inputs = matrix_multiply(layer.weights, layer_outputs[-1])
+        end
+
         layer_outputs << apply_function(layer_inputs, ActivationFunctions.method(@activation_func))
 
         # Apply dropout to hidden layers
@@ -101,13 +111,96 @@ module Qoa
 
       # Compute weight deltas
       weight_deltas = []
-      @layers.each_with_index do |_, i|
+      @layers.each_with_index do |layer, i|
         gradients = matrix_multiply_element_wise(errors[i], apply_function(layer_outputs[i + 1], ActivationFunctions.method(derivative_func)))
-        w_delta = matrix_multiply(gradients, transpose(layer_outputs[i]))
+        if layer.is_a?(Qoa::Layers::ConvolutionalLayer)
+          w_delta = conv_weight_delta(layer, gradients, layer_outputs[i])
+        elsif layer.is_a?(Qoa::Layers::PoolingLayer)
+          w_delta = pool_weight_delta(layer, gradients, layer_outputs[i])
+        else
+          w_delta = matrix_multiply(gradients, transpose(layer_outputs[i]))
+        end
         weight_deltas << w_delta
       end
 
       weight_deltas
     end
+
+    def calculate_regularization_penalty(weights, l1_lambda, l2_lambda)
+      l1_penalty = weights.map do |row|
+        row.nil? ? nil : row.map { |x| x.nil? ? nil : (x < 0 ? -1 : 1) }
+      end
+      l1_penalty = scalar_multiply(l1_lambda, l1_penalty)
+
+      l2_penalty = scalar_multiply(l2_lambda, weights)
+
+      matrix_add(l1_penalty, l2_penalty)
+    end
+
+    def convolution(layer, inputs)
+      output_size = layer.output_size
+      kernel_size = layer.kernel_size
+      stride = layer.stride
+
+      output = Array.new(output_size) { Array.new(inputs.length - kernel_size + 1) }
+      layer.weights.each_with_index do |row, i|
+        inputs.each_cons(kernel_size).each_with_index do |input_slice, j|
+          output[i][j] = row.zip(input_slice).map { |a, b| a * b }.reduce(:+)
+        end
+      end
+
+      output
+    end
+
+    def pooling(layer, inputs)
+      output_size = layer.output_size
+      pool_size = layer.pool_size
+      stride = layer.stride || 1
+
+      # Calculate the number of columns in the output array
+      output_columns = inputs[0].length - pool_size
+      output_columns = output_columns <= 0 ? 1 : ((output_columns) / stride.to_f).ceil + 1
+
+      output = Array.new(output_size) { Array.new(output_columns) }
+
+      (0...output_size).each do |i|
+        (0...output_columns).each do |j|
+          start_idx = j * stride
+          end_idx = start_idx + pool_size - 1
+          next if inputs[i].nil? || inputs[i].length < end_idx + 1 # Add this check to avoid accessing an index that does not exist
+          pool_slice = inputs[i].slice(start_idx..end_idx)
+          output[i][j] = pool_slice.max
+        end
+      end
+
+      output
+    end
+
+    def conv_weight_delta(layer, gradients, inputs)
+      kernel_size = layer.kernel_size
+      stride = layer.stride
+
+      deltas = layer.weights.map do |row|
+        inputs.each_cons(kernel_size).map do |input_slice|
+          row.zip(input_slice).map { |a, b| a * b }.reduce(:+)
+        end
+      end
+
+      deltas
+    end
+
+    def pool_weight_delta(layer, gradients, inputs)
+      pool_size = layer.pool_size
+      stride = layer.stride
+
+      deltas = inputs.each_slice(stride).map do |input_slice|
+        input_slice.each_cons(pool_size).map do |pool_slice|
+          max_index = pool_slice.each_with_index.max[1]
+          pool_slice.map.with_index { |v, i| (i == max_index) ? v * gradients[i] : 0 }
+        end
+      end
+
+      deltas
+    end
   end
 end

data/lib/qoa/utils.rb

@@ -34,16 +34,10 @@ module Qoa
       @epsilon = model_data[:epsilon]
       @batch_size = model_data[:batch_size]
 
-      @layers = model_data[:weights].map { |w| Layer.new(w.first.size, w.size) }
+      @layers = model_data[:weights].map { |w| Qoa::Layers::Layer.new(w.first.size, w.size) }
       @layers.each_with_index do |layer, i|
         layer.weights = model_data[:weights][i]
       end
     end
-
-    def mean_squared_error(prediction, target)
-      raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
-
-      prediction.zip(target).map { |p, t| (p - t) ** 2 }.sum / prediction.size
-    end
   end
 end

data/lib/qoa/version.rb

@@ -1,3 +1,3 @@
 module Qoa
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qoa
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Daniel M. Matongo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-01 00:00:00.000000000 Z
+date: 2023-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 description: Qoa is a simple machine learning library for Ruby, including a basic
   feedforward neural network implementation with backpropagation.
 email:
@@ -65,7 +79,11 @@ files:
 - code_of_conduct.md
 - lib/qoa.rb
 - lib/qoa/activation_functions.rb
-- lib/qoa/layer.rb
+- lib/qoa/err/validations.rb
+- lib/qoa/layers/convolutional_layer.rb
+- lib/qoa/layers/layer.rb
+- lib/qoa/layers/pooling_layer.rb
+- lib/qoa/loss_functions.rb
 - lib/qoa/matrix_helpers.rb
 - lib/qoa/neural_network.rb
 - lib/qoa/training.rb

data/lib/qoa/layer.rb

@@ -1,20 +0,0 @@
-module Qoa
-  class Layer
-    attr_reader :input_size, :output_size, :weights
-
-    def initialize(input_size, output_size)
-      @input_size = input_size
-      @output_size = output_size
-      @weights = random_matrix(output_size, input_size)
-    end
-
-    def random_matrix(rows, cols)
-      limit = Math.sqrt(6.0 / (rows + cols))
-      Array.new(rows) { Array.new(cols) { rand(-limit..limit) } }
-    end
-
-    def weights=(new_weights)
-      @weights = new_weights
-    end
-  end
-end