rubygrad 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 557100157146af78688b97ea078e871fe7caf7fcac573dbaf1df072bdcdc0f43
+  data.tar.gz: 44b62c4a04b6ef83f7a5a6d21df1ae97364d7b604169fd76de9259512de9aebc
+SHA512:
+  metadata.gz: fdd16f9646f97c678dcf6239bc19e228767f01b02ec0456875dfa53a0dc27f0d218e20a3565840c13262efd46036d1589157ffb3ef6c1e4727d16a36688e3e86
+  data.tar.gz: 643ea4365dfc79de6101a57bd7750e925f8c659bb42c4706fa64542ddb64e246fb9c90549f450d08015043c904ef818e59c3a6c5dacb939248a68cfed01bdf2f

data/lib/nn.rb

@@ -0,0 +1,156 @@
+require_relative "value.rb"
+
+class Neuron
+
+    def initialize(number_of_inputs)
+        @initial_weights = Array.new(number_of_inputs) { rand(-1.0..1.0) }
+        @initial_bias = rand(-1.0..1.0)
+
+        @weights = @initial_weights.map { |w| Value.new(w) }
+        @bias = Value.new(@initial_bias)
+    end
+
+    def reset_params
+        @initial_weights.each_with_index do |w,i|
+            @weights[i].value = w
+        end
+        @bias.value = @initial_bias
+    end
+
+    def set_params(params)
+        n = 1 + @weights.size
+        raise "Illegal number of parameters: #{params.size} expected #{n}" if n != params.size
+        @bias.value = params[0]
+        (1...params.size).each { |i| @weights[i - 1].value = params[i] }
+    end
+
+    attr_reader :weights, :bias
+
+    def parameters
+        self.weights + [self.bias]
+    end
+    
+    def calc(inputs, activation)
+        # xw + b
+        n = self.weights.size
+        raise "Wrong number of inputs! #{inputs.size} expected #{n}" unless n == inputs.size
+        sum = self.bias
+        n.times do |index|
+            sum += self.weights[index] * inputs[index]
+        end
+        if activation == :tanh
+            sum.tanh
+        elsif activation == :relu
+            sum.relu
+        elsif activation == :sigmoid
+            sum.sigmoid
+        else
+            raise "Unsupported activation function: #{activation}"
+        end
+    end
+end
+
+class Layer
+
+    def initialize(number_of_inputs, number_of_outputs)
+        @neurons = Array.new(number_of_outputs) { Neuron.new(number_of_inputs) }
+    end
+
+    attr_reader :neurons
+
+    def parameters
+        params = []
+        self.neurons.each { |n| params += n.parameters }
+        params
+    end
+
+    def reset_params
+        self.neurons.each { |n| n.reset_params }
+    end
+
+    def calc(inputs, activation)
+        outs = []
+        self.neurons.each do |neuron|
+            outs << neuron.calc(inputs, activation)
+        end
+        outs
+    end
+end
+
+class MLP 
+    
+    def initialize(*layers_config)
+        number_of_layers = layers_config.size
+        @layers = Array.new(number_of_layers - 1) # input layer is not really a layer object
+        (number_of_layers - 1).times do |i|
+            @layers[i] = Layer.new(layers_config[i], layers_config[i + 1])
+        end
+        @layers_config = layers_config
+    end
+
+    attr_reader :layers
+
+    def inspect
+        "MLP(#{@layers_config.join(", ")})"
+    end
+
+    def parameters
+        params = []
+        self.layers.each { |layer| params += layer.parameters }
+        params
+    end
+
+    def show_params(in_words = false)
+        if in_words
+            n = @layers_config[0]
+            puts "Layer 0: (#{n} input#{n > 1 ? "s" : ""})"
+            self.layers.each_with_index do |layer, i|
+                n = layer.neurons.size
+                puts "Layer #{i + 1}: (#{n} neuron#{n > 1 ? "s" : ""})"
+                layer.neurons.each_with_index do |neuron, ii|
+                    n = neuron.weights.size
+                    puts "\tNeuron #{ii + 1}: (#{n} weight#{n > 1 ? "s" : ""})"
+                    puts "\t\tBias: #{neuron.bias.value}"
+                    w = neuron.weights.map { |v| v.value }.join(", ")
+                    puts "\t\tWeights: #{w}"
+                end
+            end
+        else
+            n = @layers_config[0]
+            self.layers.each_with_index do |layer, i|
+                n = layer.neurons.size
+                puts "["
+                layer.neurons.each_with_index do |neuron, ii|
+                    w = neuron.weights.map { |v| v.value }.join(", ")
+                    puts "\t[ #{neuron.bias.value}, #{w} #{ii == layer.neurons.size - 1 ? ']' : '],'}"
+                end
+                puts i == self.layers.size - 1 ? "]" : "],"
+            end
+        end
+        nil
+    end
+
+    def reset_params
+        self.layers.each { |layer| layer.reset_params }
+    end
+
+    def set_params(params)
+        params.each_with_index do |layer, li|
+            layer.each_with_index do |neuron, ni|
+                self.layers[li].neurons[ni].set_params(neuron)
+            end
+        end
+    end
+
+    def zero_grad
+        self.parameters.each { |p| p.grad = 0.0 }
+    end
+
+    def calc(inputs, activation)
+        out = inputs
+        self.layers.each do |layer|
+            out = layer.calc(out, activation) # chain the results forward, layer by layer
+        end
+        out.size == 1 ? out[0] : out # for convenience
+    end
+end

data/lib/test_coerce.rb

@@ -0,0 +1,79 @@
+
+class MyValue
+
+    def initialize(value)
+        @value = value
+    end
+
+    attr_reader :value
+
+    private def to_v(other) = other.is_a?(MyValue) ? other : MyValue.new(other)
+
+    def +(other)
+        other = to_v(other)
+        out = MyValue.new(self.value + other.value)
+    end
+
+    def *(other)
+        other = to_v(other)
+        out = MyValue.new(self.value * other.value)
+    end
+
+    def **(other)
+        out = MyValue.new(self.value ** other)
+    end
+
+    def -@
+        self * -1
+    end
+
+    def -(other) 
+        self + (-other)
+    end
+
+    def /(other)
+        self * (other ** -1)
+    end
+
+    def coerce(other)
+        other = to_v(other)
+        [other, self.value]
+    end
+
+    def to_s
+        value.to_s
+    end
+
+end
+
+a = MyValue.new(4.0) # a MyValue
+b = MyValue.new(2.0) # a MyValue
+c = 2.0 # an Integer 
+
+res = a - b
+puts "#{res} is #{res.class}" # => 2.0 is MyValue
+
+res = b - a
+puts "#{res} is #{res.class}" # => -2.0 is MyValue
+
+res = a - c
+puts "#{res} is #{res.class}" # => 2.0 is MyValue
+
+res = c - a
+puts "#{res} is #{res.class}" # => -2.0 is MyValue
+
+puts
+
+res = a / b
+puts "#{res} is #{res.class}" # => 2.0 is MyValue
+
+res = b / a
+puts "#{res} is #{res.class}" # => 0.5 is MyValue
+
+res = a / c
+puts "#{res} is #{res.class}" # => 2.0 is MyValue
+
+res = c / a
+puts "#{res} is #{res.class}" # => 0.5 is MyValue
+
+

data/lib/value.rb

@@ -0,0 +1,164 @@
+require 'set'
+
+class Value
+
+    def initialize(value, prev = [])
+        @value = value
+        @grad = 0
+        @prev = prev.uniq.freeze
+        @calc_gradient = lambda { }
+    end
+
+    attr_reader :value, :grad, :prev, :calc_gradient
+    attr_writer :calc_gradient, :grad, :value
+
+    def +(other)
+        other = to_v(other)
+        out = Value.new(self.value + other.value, [self, other])
+
+        out.calc_gradient = lambda do
+            self.grad += out.grad
+            other.grad += out.grad
+        end
+
+        return out
+    end
+
+    def *(other)
+        other = to_v(other)
+        out = Value.new(self.value * other.value, [self, other])
+
+        out.calc_gradient = lambda do
+            self.grad += other.value * out.grad
+            other.grad += self.value * out.grad
+        end
+
+        return out
+    end
+
+    def **(other)
+        out = Value.new(self.value ** other, [self])
+
+        out.calc_gradient = lambda do
+            self.grad += (other * self.value ** (other - 1)) * out.grad
+        end
+
+        return out
+    end
+
+    def tanh
+        t = (Math.exp(2.0 * self.value) - 1.0) / (Math.exp(2.0 * self.value) + 1.0)
+        out = Value.new(t, [self])
+
+        out.calc_gradient = lambda do
+            self.grad += (1.0 - t ** 2.0) * out.grad
+        end
+
+        return out
+    end
+
+    def sigmoid
+        e = Math.exp(-1.0 * self.value)
+        t = 1.0 / (1.0 + e)
+        out = Value.new(t, [self])
+        
+        out.calc_gradient = lambda do
+            self.grad += t * (1.0 - t) * out.grad
+        end
+
+        return out
+    end
+
+    def relu
+        n = self.value < 0 ? 0.0 : self.value
+        out = Value.new(n, [self])
+
+        out.calc_gradient = lambda do
+            self.grad += (out.value > 0 ? 1.0 : 0.0) * out.grad
+        end
+
+        return out
+    end
+
+    def exp 
+        out = Value.new(Math.exp(self.value), [self])
+
+        out.calc_gradient = lambda do
+            self.grad += out.value * out.grad
+        end
+
+        return out
+    end
+
+    def -@
+        self * -1
+    end
+
+    def -(other)
+        self + (-other)
+    end
+
+    def /(other)
+        self * (other ** -1)
+    end
+
+    def coerce(other)
+        other = to_v(other)
+        [other, self.value]
+    end
+
+    def build_topo_graph(start)
+        topo = []
+        visited = Set.new
+        build_topo = lambda do |v|
+            if !visited.include?(v)
+                visited.add(v)
+                v.prev.each do |child|
+                    build_topo.call(child)
+                end
+                topo.append(v)
+            end
+        end
+        build_topo.call(start)
+        return topo
+    end
+
+    def backward
+        topo = build_topo_graph(self)
+        self.grad = 1.0
+        topo.reverse_each do |node|
+            node.calc_gradient.call
+        end
+    end
+
+    def to_s
+        value.to_s
+    end
+
+    def inspect
+        "Value(value=#{value}, grad=#{grad})"
+    end
+
+    private def to_v(other) = other.is_a?(Value) ? other : Value.new(other)
+
+end
+
+
+=begin
+x1 = Value.new(2.0)
+x2 = Value.new(0.0)
+w1 = Value.new(-3.0)
+w2 = Value.new(1.0)
+b = Value.new(6.881373587)
+x1w1 = x1 * w1
+x2w2 = x2 * w2
+x1w1x2w2 = x1w1 + x2w2
+n = x1w1x2w2 + b
+o = n.tanh
+o.backward
+
+puts x1.inspect,x2.inspect,w1.inspect,w2.inspect
+
+=end
+
+

data/mlp_example.rb

@@ -0,0 +1,43 @@
+require_relative 'lib/nn.rb'
+
+nn = MLP.new(3, 4, 4, 1)
+
+x_inputs = [
+    [2.0, 3.0, -1.0],
+    [3.0, -1.0, 0.5],
+    [0.5, 1.0, 1.0],
+    [1.0, 1.0, -1.0]
+]
+y_expected = [1.0, -1.0, -1.0, 1.0] # desired
+
+passes = 2000
+learning_rate = 0.2
+
+_loss_precision = 10
+_passes_format = "%#{passes.digits.length}d"
+_loss_format = "%.#{_loss_precision}f"
+
+(0...passes).each do |pass| 
+
+    # forward pass (calculate output)
+    y_calculated = x_inputs.map { |x| nn.calc(x, :tanh) }
+
+    # loss function (check how good the neural net is)
+    loss = 0.0
+    y_expected.each_index { |i| loss += (y_calculated[i] - y_expected[i]) ** 2 }
+
+    # backward pass (calculate gradients)
+    nn.zero_grad
+    loss.backward
+
+    # improve neural net (update weights and biases)
+    nn.parameters.each { |p| p.value -= learning_rate * p.grad }
+
+    puts "Pass #{_passes_format % (pass + 1)} => Learning rate: #{"%.10f" % learning_rate} => Loss: #{_loss_format % loss.value}" if (pass + 1) % 100 == 0 or pass == 0
+
+    break if loss.value == 0 # just for fun and just in case
+end
+
+y_calculated = x_inputs.map { |x| nn.calc(x, :tanh) }
+puts
+puts y_calculated

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: rubygrad
+version: !ruby/object:Gem::Version
+  version: 1.1.1
+platform: ruby
+authors:
+- Sergio Oliveira Jr
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description:
+email: sergio.oliveira.jr@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/nn.rb
+- lib/test_coerce.rb
+- lib/value.rb
+- mlp_example.rb
+homepage: https://github.com/saoj/rubygrad
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.7
+signing_key:
+specification_version: 4
+summary: A port of Andrej Karpathy's micrograd to Ruby.
+test_files: []