qoa 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ca1c9d06ebd0907eb52f52d4a34c1f44e53b8674efda61731d5ff397f2650a6
-  data.tar.gz: cd31fa3f0bafef6c043d7b22b368ca84c6a1240ce2c782d058d9c032d6c16be9
+  metadata.gz: a45d057fc9994389ef429aa82b3207a2e30b8e67a880f3ad6e6b57f7e7a79b75
+  data.tar.gz: 273f73d4184545b75ec338f75c9ea39b5e4e2cc6e2582091bab036e93f06b320
 SHA512:
-  metadata.gz: d4d293ad9e455a5f9017583ba5d78e717210900226eb72f776d69fbad6d7ac16b99abdaf956707310f11b02b60e2bc78a43118ed84045b5cda7a115d0936f143
-  data.tar.gz: 257b167c2aaa3e9206c67b5c3ff3bee78a1d29aa355a5c64e43cd091c7edd3bec17a435105947a1001424d43521baa35e503bd5c02c7d7927fbebfb50a78f153
+  metadata.gz: 00eedcfdee31e73c559be6b8c3e75d31c72d2f802a40e9322c82b0bfe6e615ce9f2498d559722fa360d9ebb7d23328f8275c94374b40651d82e3705f869e413a
+  data.tar.gz: c125116a1e5deb7d345315eaa9954e006b198a881864b123b452906dcd1d009f473bbfc82a7cd7f09a67dca7c61de83aefb8156eed4cfde8b5d593858448c017

data/README.md

@@ -13,12 +13,18 @@ Qoa is a simple and customizable neural network library for Ruby. It allows you
 
 ## Installation
 
-Simply copy the `neural_network.rb`, `activation_functions.rb`, and `matrix_helpers.rb` files into your project and require them.
+### Install via RubyGems
+
+You can install the gem via RubyGems:
+
+```
+gem install qoa
+```
+
+Then, require the gem in your project:
 
 ```ruby
-require_relative 'neural_network'
-require_relative 'activation_functions'
-require_relative 'matrix_helpers'
+require 'qoa'
 ```
 
 ## Usage
@@ -40,7 +46,30 @@ To create a new neural network, you can initialize an instance of `Qoa::NeuralNe
 Example:
 
 ```ruby
-nn = Qoa::NeuralNetwork.new(784, [128, 64], 10, 0.001, 0.5, :relu, 0.9, 1e-8, 32)
+require 'qoa'
+
+input_nodes = 784 # Number of input features (e.g., 28x28 pixels for MNIST dataset)
+hidden_layers = [128, 64] # Two hidden layers with 128 and 64 nodes each
+output_nodes = 10 # Number of output classes (e.g., 10 for MNIST dataset)
+learning_rate = 0.01
+dropout_rate = 0.5
+activation_func = :relu
+
+nn = Qoa::NeuralNetwork.new(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func)
+```
+
+### Saving and Loading Models
+
+To save the trained model to a file, call the `save_model` method:
+
+```ruby
+nn.save_model('model.json')
+```
+
+To load a previously saved model, call the `load_model` method:
+
+```ruby
+nn.load_model('model.json')
 ```
 
 ### Training the Neural Network
@@ -98,4 +127,4 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/mmaton
 
 ## License
 
-The library is available as open source under the terms of the [MIT License](http://opensource.org/licenses/Apache-2.0).
+The library is available as open source under the terms of the [Apache-2.0 License](http://opensource.org/licenses/Apache-2.0).

data/lib/qoa/layer.rb

@@ -0,0 +1,20 @@
+module Qoa
+  class Layer
+    attr_reader :input_size, :output_size, :weights
+
+    def initialize(input_size, output_size)
+      @input_size = input_size
+      @output_size = output_size
+      @weights = random_matrix(output_size, input_size)
+    end
+
+    def random_matrix(rows, cols)
+      limit = Math.sqrt(6.0 / (rows + cols))
+      Array.new(rows) { Array.new(cols) { rand(-limit..limit) } }
+    end
+
+    def weights=(new_weights)
+      @weights = new_weights
+    end
+  end
+end

data/lib/qoa/neural_network.rb

@@ -1,10 +1,12 @@
-require 'concurrent'
+require_relative 'layer'
 require_relative 'activation_functions'
-require_relative 'matrix_helpers'
+require_relative 'training'
+require_relative 'utils'
 
 module Qoa
   class NeuralNetwork
-    include MatrixHelpers
+    include Training
+    include Utils
     attr_reader :input_nodes, :hidden_layers, :output_nodes, :learning_rate, :activation_func, :dropout_rate, :decay_rate, :epsilon, :batch_size
 
     def initialize(input_nodes, hidden_layers, output_nodes, learning_rate, dropout_rate, activation_func = :sigmoid, decay_rate = 0.9, epsilon = 1e-8, batch_size = 10)
@@ -18,101 +20,29 @@ module Qoa
       @epsilon = epsilon
       @batch_size = batch_size
 
-      @weights = []
-      @weights << random_matrix(hidden_layers[0], input_nodes)
+      @layers = []
+      @layers << Layer.new(input_nodes, hidden_layers[0])
       hidden_layers.each_cons(2) do |l1, l2|
-        @weights << random_matrix(l2, l1)
+        @layers << Layer.new(l1, l2)
       end
-      @weights << random_matrix(output_nodes, hidden_layers[-1])
+      @layers << Layer.new(hidden_layers[-1], output_nodes)
     end
 
-    def random_matrix(rows, cols)
-      limit = Math.sqrt(6.0 / (rows + cols))
-      Array.new(rows) { Array.new(cols) { rand(-limit..limit) } }
+    def query(inputs)
+      layer_outputs = forward_pass(inputs)
+      layer_outputs.last.flatten
     end
 
-    def train(inputs, targets)
+    def calculate_loss(inputs, targets)
       raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
 
-      inputs.zip(targets).each_slice(@batch_size) do |batch|
-        train_batch(batch)
+      total_loss = 0.0
+      inputs.zip(targets).each do |input, target|
+        prediction = query(input)
+        total_loss += mean_squared_error(prediction, target)
       end
-    end
-
-    def train_batch(batch)
-      derivative_func = "#{@activation_func}_derivative"
-      batch_inputs = batch.map { |x, _| x }
-      batch_targets = batch.map { |_, y| y }
-
-      # Forward pass
-      layer_outputs = batch_inputs.map { |inputs| forward_pass(inputs) }
-
-      # Backward pass
-      # Using thread pool to parallelize the backward pass for each input in the batch
-      pool = Concurrent::FixedThreadPool.new(4)
-      weight_deltas = Array.new(@weights.size) { Array.new(@weights[0].size) { Array.new(@weights[0][0].size, 0) } }
-      mutex = Mutex.new
-
-      batch.zip(layer_outputs).each do |(inputs, targets), outputs|
-        pool.post do
-          deltas = backward_pass(inputs, targets, outputs)
-          mutex.synchronize do
-            @weights.each_with_index do |_, i|
-              weight_deltas[i] = matrix_add(weight_deltas[i], deltas[i])
-            end
-          end
-        end
-      end
-
-      pool.shutdown
-      pool.wait_for_termination
-
-      # Update weights
-      @weights.each_with_index do |w, i|
-        @weights[i] = matrix_add(w, scalar_multiply(@learning_rate / batch.size, weight_deltas[i]))
-      end
-    end
-
-    def forward_pass(inputs)
-      inputs = inputs.map { |x| [x] } # Convert to column vector
-
-      layer_outputs = [inputs]
-      @weights.each_with_index do |w, i|
-        layer_inputs = matrix_multiply(w, layer_outputs[-1])
-        layer_outputs << apply_function(layer_inputs, ActivationFunctions.method(@activation_func))
 
-        # Apply dropout to hidden layers
-        layer_outputs[-1] = apply_dropout(layer_outputs[-1], @dropout_rate) if i < @weights.size - 1
-      end
-
-      layer_outputs
-    end
-
-    def backward_pass(inputs, targets, layer_outputs)
-      derivative_func = "#{@activation_func}_derivative"
-      inputs = inputs.map { |x| [x] } # Convert to column vector
-      targets = targets.map { |x| [x] } # Convert to column vector
-
-      # Compute errors
-      errors = [matrix_subtract(targets, layer_outputs.last)]
-      (@weights.size - 1).downto(1) do |i|
-        errors << matrix_multiply(transpose(@weights[i]), errors.last)
-      end
-
-      # Compute weight deltas
-      weight_deltas = []
-      @weights.each_with_index do |w, i|
-        gradients = matrix_multiply_element_wise(errors[i], apply_function(layer_outputs[i + 1], ActivationFunctions.method(derivative_func)))
-        w_delta = matrix_multiply(gradients, transpose(layer_outputs[i]))
-        weight_deltas << w_delta
-      end
-
-      weight_deltas
-    end
-
-    def query(inputs)
-      layer_outputs = forward_pass(inputs)
-      layer_outputs.last.flatten
+      total_loss / inputs.size
     end
   end
 end

data/lib/qoa/training.rb

@@ -0,0 +1,113 @@
+require_relative 'matrix_helpers'
+require 'concurrent'
+
+module Qoa
+  module Training
+    include MatrixHelpers
+
+    def train(inputs, targets)
+      raise ArgumentError, 'inputs and targets must have the same length' if inputs.size != targets.size
+
+      inputs.zip(targets).each_slice(@batch_size) do |batch|
+        train_batch(batch)
+      end
+    end
+
+    def train_batch(batch)
+      derivative_func = "#{@activation_func}_derivative"
+      batch_inputs = batch.map { |x, _| x }
+      batch_targets = batch.map { |_, y| y }
+
+      # Forward pass
+      layer_outputs = batch_inputs.map { |inputs| forward_pass(inputs) }
+
+      # Backward pass
+      # Using thread pool to parallelize the backward pass for each input in the batch
+      pool = Concurrent::FixedThreadPool.new(4)
+      #   weight_deltas = Array.new(@layers.size - 1) { |i| Array.new(@layers[i].output_size) { Array.new(@layers[i].input_size, 0) } }
+      weight_deltas = Array.new(@layers.size) { |i| Array.new(@layers[i].output_size) { Array.new(@layers[i].input_size, 0) } }
+      mutex = Mutex.new
+
+      batch.zip(layer_outputs).each do |(inputs, targets), outputs|
+        pool.post do
+          deltas = backward_pass(inputs, targets, outputs)
+          mutex.synchronize do
+            @layers.each_with_index do |_, i|
+              weight_deltas[i] = matrix_add(weight_deltas[i], deltas[i])
+            end
+          end
+        end
+      end
+
+      pool.shutdown
+      pool.wait_for_termination
+
+      # Update weights
+      @layers.each_with_index do |layer, i|
+        layer.weights = matrix_add(layer.weights, scalar_multiply(@learning_rate / batch.size, weight_deltas[i]))
+      end
+    end
+
+    def train_with_early_stopping(inputs, targets, validation_inputs, validation_targets, max_epochs, patience)
+      best_validation_loss = Float::INFINITY
+      patience_left = patience
+      epoch = 0
+
+      while epoch < max_epochs && patience_left > 0
+        train(inputs, targets)
+        validation_loss = calculate_loss(validation_inputs, validation_targets)
+        puts "Epoch #{epoch + 1}: Validation loss = #{validation_loss}"
+
+        if validation_loss < best_validation_loss
+          best_validation_loss = validation_loss
+          save_model('best_model.json')
+          patience_left = patience
+        else
+          patience_left -= 1
+        end
+
+        epoch += 1
+      end
+
+      puts "Training stopped. Best validation loss = #{best_validation_loss}"
+      load_model('best_model.json')
+    end
+
+    def forward_pass(inputs)
+      inputs = inputs.map { |x| [x] } # Convert to column vector
+
+      layer_outputs = [inputs]
+      @layers.map(&:weights).each_with_index do |w, i|
+        layer_inputs = matrix_multiply(w, layer_outputs[-1])
+        layer_outputs << apply_function(layer_inputs, ActivationFunctions.method(@activation_func))
+
+        # Apply dropout to hidden layers
+        layer_outputs[-1] = apply_dropout(layer_outputs[-1], @dropout_rate) if i < @layers.size - 2
+      end
+
+      layer_outputs
+    end
+
+    def backward_pass(inputs, targets, layer_outputs)
+      derivative_func = "#{@activation_func}_derivative"
+      inputs = inputs.map { |x| [x] } # Convert to column vector
+      targets = targets.map { |x| [x] } # Convert to column vector
+
+      # Compute errors
+      errors = [matrix_subtract(targets, layer_outputs.last)]
+      (@layers.size - 2).downto(0) do |i|
+        errors << matrix_multiply(transpose(@layers[i + 1].weights), errors.last)
+      end
+
+      # Compute weight deltas
+      weight_deltas = []
+      @layers.each_with_index do |_, i|
+        gradients = matrix_multiply_element_wise(errors[i], apply_function(layer_outputs[i + 1], ActivationFunctions.method(derivative_func)))
+        w_delta = matrix_multiply(gradients, transpose(layer_outputs[i]))
+        weight_deltas << w_delta
+      end
+
+      weight_deltas
+    end
+  end
+end

data/lib/qoa/utils.rb

@@ -0,0 +1,49 @@
+require 'json'
+
+module Qoa
+  module Utils
+    def save_model(file_path)
+      model_data = {
+        input_nodes: @input_nodes,
+        hidden_layers: @hidden_layers,
+        output_nodes: @output_nodes,
+        learning_rate: @learning_rate,
+        activation_func: @activation_func,
+        dropout_rate: @dropout_rate,
+        decay_rate: @decay_rate,
+        epsilon: @epsilon,
+        batch_size: @batch_size,
+        weights: @layers.map(&:weights),
+      }
+
+      File.open(file_path, 'w') do |f|
+        f.write(JSON.pretty_generate(model_data))
+      end
+    end
+
+    def load_model(file_path)
+      model_data = JSON.parse(File.read(file_path), symbolize_names: true)
+
+      @input_nodes = model_data[:input_nodes]
+      @hidden_layers = model_data[:hidden_layers]
+      @output_nodes = model_data[:output_nodes]
+      @learning_rate = model_data[:learning_rate]
+      @activation_func = model_data[:activation_func].to_sym
+      @dropout_rate = model_data[:dropout_rate]
+      @decay_rate = model_data[:decay_rate]
+      @epsilon = model_data[:epsilon]
+      @batch_size = model_data[:batch_size]
+
+      @layers = model_data[:weights].map { |w| Layer.new(w.first.size, w.size) }
+      @layers.each_with_index do |layer, i|
+        layer.weights = model_data[:weights][i]
+      end
+    end
+
+    def mean_squared_error(prediction, target)
+      raise ArgumentError, 'prediction and target must have the same length' if prediction.size != target.size
+
+      prediction.zip(target).map { |p, t| (p - t) ** 2 }.sum / prediction.size
+    end
+  end
+end

data/lib/qoa/version.rb

@@ -1,3 +1,3 @@
 module Qoa
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qoa
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Daniel M. Matongo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-29 00:00:00.000000000 Z
+date: 2023-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -65,8 +65,11 @@ files:
 - code_of_conduct.md
 - lib/qoa.rb
 - lib/qoa/activation_functions.rb
+- lib/qoa/layer.rb
 - lib/qoa/matrix_helpers.rb
 - lib/qoa/neural_network.rb
+- lib/qoa/training.rb
+- lib/qoa/utils.rb
 - lib/qoa/version.rb
 homepage: https://github.com/mmatongo/qoa
 licenses: