neuronet 7.0.230416 → 8.0.251113

data/lib/neuronet/layer.rb

@@ -1,111 +1,41 @@
 # frozen_string_literal: true
 
-# Neuronet module
 module Neuronet
-  # Layer is an array of neurons.
-  class Layer < Array
-    # Length is the number of neurons in the layer.
-    def initialize(length)
-      super(length) { Neuron.new }
-    end
-
-    # This is where one enters the "real world" inputs.
-    def set(inputs)
-      0.upto(length - 1) { self[_1].value = inputs[_1] || 0.0 }
-      self
-    end
-
-    # Returns the real world values: [value, ...]
-    def values
-      map(&:value)
-    end
-
-    # Allows one to fully connect layers.
-    def connect(layer = Layer.new(length), weights: [])
-      # creates the neuron matrix...
-      each_with_index do |neuron, i|
-        weight = weights[i] || 0.0
-        layer.each { neuron.connect(_1, weight:) }
-      end
-      # The layer is returned for chaining.
-      layer
-    end
-
-    # Set layer to mirror input:
-    #   bias   = BZERO.
-    #   weight = WONE
-    # Input should be the same size as the layer.  One can set sign to -1 to
-    # anti-mirror.  One can set sign to other than |1| to scale.
-    def mirror(sign = 1)
-      each_with_index do |neuron, index|
-        neuron.bias = sign * Neuronet.bzero
-        neuron.connections[index].weight = sign * Neuronet.wone
-      end
-    end
+  # Layer is a collection of neurons with array-like behavior.
+  class Layer
+    # [LayerPresets](layer_presets.rb)
+    include LayerPresets
+    # [Arrayable](arrayable.rb)
+    include Arrayable
 
-    # Doubles up the input mirroring and anti-mirroring it.  The layer should be
-    # twice the size of the input.
-    def antithesis
-      sign = 1
-      each_with_index do |n, i|
-        n.connections[i / 2].weight = sign * Neuronet.wone
-        n.bias = sign * Neuronet.bzero
-        sign = -sign
-      end
+    # Creates layer with `length` number of neurons.
+    def initialize(length, full_neuron: Neuron)
+      @layer = Array.new(length) { full_neuron.new }
+      @endex = length - 1
     end
 
-    # Sums two corresponding input neurons above each neuron in the layer.
-    # Input should be twice the size of the layer.
-    def synthesis
-      semi = Neuronet.wone / 2
-      each_with_index do |n, i|
-        j = i * 2
-        c = n.connections
-        n.bias = Neuronet.bzero
-        c[j].weight = semi
-        c[j + 1].weight = semi
-      end
+    # Set each neuron's activation from values array.
+    # Allows the layer to be used as an input layer.
+    def set(values)
+      0.upto(@endex) { @layer[it].set values[it] }
     end
 
-    # Set layer to average input.
-    def average(sign = 1)
-      bias = sign * Neuronet.bzero
-      each do |n|
-        n.bias = bias
-        weight = sign * Neuronet.wone / n.connections.length
-        n.connections.each { _1.weight = weight }
-      end
-    end
+    # For each neuron in the layer, updates the neuron's activation.
+    def update = @layer.each(&:update)
 
-    # updates layer with current values of the previous layer
-    def partial
-      each(&:partial)
-    end
-
-    def average_mju
-      Neuronet.learning * sum { Neuron.mju(_1) } / length
-    end
-
-    # Takes the real world target for each neuron in this layer and
-    # backpropagates the error to each neuron.
-    def train(target, mju = nil)
-      0.upto(length - 1) do |index|
-        neuron = self[index]
-        error = (target[index] - neuron.value) /
-                (mju || (Neuronet.learning * Neuron.mju(neuron)))
-        neuron.backpropagate(error)
+    # Fully connects this layer to another.
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
       end
-      self
     end
 
-    # Layer inspects as "label:value,..."
-    def inspect
-      map(&:inspect).join(',')
+    # Raw pre-squashed values of all neurons in the layer.
+    # Allows the layer to be use as an output layer.
+    def values
+      @layer.map(&:value)
     end
 
-    # Layer puts as "label,..."
-    def to_s
-      map(&:to_s).join(',')
-    end
+    def to_a = @layer
   end
 end

data/lib/neuronet/layer_presets.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # LayerPresets initializes layer weights/biases for interpret-able functions.
+  module LayerPresets
+    BZERO = 0.5 / (0.5 - Squash.squash(1.0))
+    WONE = -2.0 * BZERO
+
+    # Set layer to roughly mirror it's input.
+    # Input should be the same size as the layer.
+    def mirror(sign = 1.0)
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        neuron.connections[index].weight = sign * WONE
+      end
+    end
+
+    # Doubles up the input both mirroring and anti-mirroring it.
+    # The layer should be twice the size of the input.
+    def antithesis
+      sign = 1.0
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        neuron.connections[index / 2].weight = sign * WONE
+        sign = -sign
+      end
+    end
+
+    # Sums two corresponding input neurons above each neuron in the layer.
+    # Input should be twice the size of the layer.
+    def synthesis(sign = 1.0)
+      semi = sign * WONE / 2.0
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        j = index * 2
+        connections = neuron.connections
+        connections[j].weight = semi
+        connections[j + 1].weight = semi
+      end
+    end
+
+    # Set layer to average input.
+    def average(sign = 1.0)
+      bias = sign * BZERO
+      each do |neuron|
+        neuron.bias = bias
+        connections = neuron.connections
+        weight = sign * WONE / connections.size
+        connections.each { it.weight = weight }
+      end
+    end
+  end
+end

data/lib/neuronet/middle_layer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Neuron Layer
+  class MiddleLayer
+    include LayerPresets
+    include Arrayable
+
+    def initialize(length, middle_neuron: MiddleNeuron)
+      @layer = Array.new(length) { middle_neuron.new }
+    end
+
+    def update = @layer.each(&:update)
+
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
+      end
+    end
+
+    def to_a = @layer
+  end
+end

data/lib/neuronet/middle_neuron.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class MiddleNeuron
+    include NeuronStats
+    include Backpropagate
+    include Squash
+
+    def initialize
+      @activation  = 0.5
+      @bias        = 0.0
+      @connections = []
+    end
+
+    attr_accessor :bias
+    attr_reader   :activation, :connections
+
+    def connect(neuron, weight = 0.0)
+      @connections << Connection.new(neuron, weight)
+    end
+
+    def value
+      @bias + @connections.sum(&:value)
+    end
+
+    def update
+      @activation = squash(value)
+    end
+  end
+end

data/lib/neuronet/mlp.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Multi Layer Perceptron(3 layers)
+  class MLP
+    include NetworkStats
+    include Exportable
+    include Trainable
+    include Arrayable
+
+    # rubocop: disable Metrics
+    def initialize(input_size, middle_size, output_size,
+                   input_neuron: InputNeuron,
+                   middle_neuron: MiddleNeuron,
+                   output_neuron: OutputNeuron)
+      @input_layer = InputLayer.new(input_size, input_neuron:)
+      @middle_layer = MiddleLayer.new(middle_size, middle_neuron:)
+      @output_layer = OutputLayer.new(output_size, output_neuron:)
+      @middle_layer.connect(@input_layer)
+      @output_layer.connect(@middle_layer)
+    end
+    # rubocop: enable Metrics
+
+    attr_reader :input_layer, :middle_layer, :output_layer
+
+    def set(values)
+      @input_layer.set(values)
+    end
+
+    def update
+      @middle_layer.update
+    end
+
+    def values
+      @output_layer.values
+    end
+
+    def *(other)
+      set(other)
+      update
+      values
+    end
+
+    def to_a = [@input_layer, @middle_layer, @output_layer]
+  end
+end

data/lib/neuronet/network_stats.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Network Stats
+  module NetworkStats
+    # See https://github.com/carlosjhr64/neuronet/wiki
+    # |𝝂| = 𝔪 + ¼√𝑁*𝔪' + ¼√𝑁*¼√𝑁'*𝔪" + ...
+    def expected_nju!
+      nju = 0.0
+      mult = 1.0
+      reverse[1..].each do |layer|
+        size = layer.size
+        mju = 1 + (0.5 * size)
+        nju += mult * mju
+        mult *= 0.25 * Math.sqrt(size)
+      end
+      @expected_nju = nju
+    end
+
+    def expected_nju
+      @expected_nju || expected_nju!
+    end
+
+    def njus
+      output_layer.map(&:nju)
+    end
+  end
+end

data/lib/neuronet/neuron.rb

@@ -1,146 +1,45 @@
 # frozen_string_literal: true
 
-# Neuronet module / Neuron class
 module Neuronet
-  # A Neuron is capable of creating connections to other neurons.  The
-  # connections attribute is a list of the neuron's connections to other
-  # neurons.  A neuron's bias is it's kicker (or deduction) to it's activation
-  # value, a sum of its connections values.
+  # Neuron represents a single node in a neural network.
+  # It holds @activation, @bias, and incoming @connections.
   class Neuron
-    # For bookkeeping, each Neuron is given a label, starting with 'a' by
-    # default.
-    class << self; attr_accessor :label; end
-    Neuron.label = 'a'
+    # [nju computation](neuron_stats.rb)
+    include NeuronStats
+    # [back-propagation of errors](backpropagate.rb)
+    include Backpropagate
+    # [squash/unsquash methods](squash.rb)
+    include Squash
 
-    attr_reader :label, :activation, :connections
-    attr_accessor :bias
-
-    # The neuron's mu is the sum of the connections' mu(activation), plus one
-    # for the bias:
-    #   𝛍 := 1+∑𝐚'
-    def mu
-      return 0.0 if @connections.empty?
-
-      1 + @connections.sum(&:mu)
-    end
-
-    # Reference the library's wiki:
-    #   𝒆ₕ ~ 𝜀(𝝁ₕ + 𝜧ₕⁱ𝝁ᵢ + 𝜧ₕⁱ𝜧ᵢʲ𝝁ⱼ + 𝜧ₕⁱ𝜧ᵢʲ𝜧ⱼᵏ𝝁ₖ + ...)
-    # 𝜧ₕⁱ𝝁ᵢ is:
-    #   neuron.mju{ |connected_neuron| connected_neuron.mu }
-    # 𝜧ₕⁱ𝜧ᵢʲ𝝁ⱼ is:
-    #   nh.mju{ |ni| ni.mju{ |nj| nj.mu }}
-    def mju(&block)
-      @connections.sum { _1.mju * block[_1.neuron] }
+    # Initializes a neuron with default activation 0.5 and zero bias.
+    def initialize
+      @activation  = 0.5
+      @bias        = 0.0
+      @connections = [] # incoming connections
     end
 
-    # Full recursive implementation of mju:
-    def self.mju(neuron)
-      return 0.0 if neuron.connections.empty?
+    attr_accessor :bias # bias is settable
+    attr_reader   :activation, :connections # activation is read-only
 
-      neuron.mu + neuron.mju { |connected_neuron| Neuron.mju(connected_neuron) }
+    # Sets activation by applying squash to raw input value.
+    def set(value)
+      @activation = squash(value)
     end
 
-    # 𝓓𝒗⌈𝒗 = (1-⌈𝒗)⌈𝒗 = (1-𝒂)𝒂 = 𝓑𝒂
-    def derivative = Neuronet.derivative[@activation]
-
-    # 𝝀 = 𝓑𝒂𝛍
-    def lamda = derivative * mu
-
-    # 𝜿 := 𝜧 𝝁' = 𝑾 𝓑𝒂'𝝁' = 𝑾 𝝀'
-    # def kappa = mju(&:mu)
-    def kappa = @connections.sum(&:kappa)
-
-    # 𝜾 := 𝜧 𝜧' 𝝁" = 𝜧 𝜿'
-    def iota = mju(&:kappa)
-
-    # One can explicitly set the neuron's value, typically used to set the input
-    # neurons.  The given "real world" value is squashed into the neuron's
-    # activation value.
-    def value=(value)
-      # If value is out of bounds, set it to the bound.
-      if value.abs > Neuronet.maxv
-        value = value.positive? ? Neuronet.maxv : -Neuronet.maxv
-      end
-      @activation = Neuronet.squash[value]
+    # Creates a weighted connection to another neuron.
+    # See [Neuronet::Connection](connection.rb)
+    def connect(neuron, weight = 0.0)
+      @connections << Connection.new(neuron, weight)
     end
 
-    # The "real world" value of the neuron is the unsquashed activation value.
-    def value = Neuronet.unsquash[@activation]
-
-    # The initialize method sets the neuron's value, bias and connections.
-    def initialize(value = 0.0, bias: 0.0, connections: [])
-      self.value   = value
-      @connections = connections
-      @bias        = bias
-      @label       = Neuron.label
-      Neuron.label = Neuron.label.next
+    # Computes(raw output)value: bias + sum of incoming connection values.
+    def value
+      @bias + @connections.sum(&:value)
     end
 
-    # Updates the activation with the current value of bias and updated values
-    # of connections.
+    # Updates activation by squashing the current value(see above).
     def update
-      return @activation if @connections.empty?
-
-      self.value = @bias + @connections.sum(&:update)
-      @activation
+      @activation = squash(value)
     end
-
-    # For when connections are already updated, Neuron#partial updates the
-    # activation with the current values of bias and connections.  It is not
-    # always necessary to burrow all the way down to the terminal input neuron
-    # to update the current neuron if it's connected neurons have all been
-    # updated.  The implementation should set it's algorithm to use partial
-    # instead of update as update will most likely needlessly update previously
-    # updated neurons.
-    def partial
-      return @activation if @connections.empty?
-
-      self.value = @bias + @connections.sum(&:partial)
-      @activation
-    end
-
-    # The backpropagate method modifies the neuron's bias in proportion to the
-    # given error and passes on this error to each of its connection's
-    # backpropagate method.  While updates flows from input to output, back-
-    # propagation of errors flows from output to input.
-    def backpropagate(error)
-      return self if @connections.empty?
-
-      @bias += Neuronet.noise[error]
-      if @bias.abs > Neuronet.maxb
-        @bias = @bias.positive? ? Neuronet.maxb : -Neuronet.maxb
-      end
-      @connections.each { |connection| connection.backpropagate(error) }
-      self
-    end
-
-    # Connects the neuron to another neuron.  The default weight=0 means there
-    # is no initial association.  The connect method is how the implementation
-    # adds a connection, the way to connect a neuron to another.  To connect
-    # "output" to "input", for example, it is:
-    #	  input = Neuronet::Neuron.new
-    #	  output = Neuronet::Neuron.new
-    #	  output.connect(input)
-    # Think "output" connects to "input".
-    def connect(neuron = Neuron.new, weight: 0.0)
-      @connections.push(Connection.new(neuron, weight:))
-      # Note that we're returning the connected neuron:
-      neuron
-    end
-
-    # Tacks on to neuron's inspect method to show the neuron's bias and
-    # connections.
-    def inspect
-      fmt = Neuronet.format
-      if @connections.empty?
-        "#{@label}:#{fmt % value}"
-      else
-        "#{@label}:#{fmt % value}|#{[(fmt % @bias), *@connections].join('+')}"
-      end
-    end
-
-    # A neuron plainly puts itself as it's label.
-    def to_s = @label
   end
 end

data/lib/neuronet/neuron_stats.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # NeuronStats provides network analysis methods.
+  module NeuronStats
+    # Returns the total tally of parameters in the downstream
+    # network subgraph from this neuron.
+    # This includes the neuron's bias (1 parameter),
+    # the weights of its incoming connections (one per connection),
+    # and the sum of parameters from all downstream neurons.
+    # Parameters from a shared neuron are counted multiple times
+    # if accessed via multiple pathways,
+    # reflecting the total parameter influence through all paths.
+    # Returns 0 for a neuron with no downstream connections.
+    def downstream_params_tally
+      return 0 if (size = connections.size).zero?
+
+      1 + size + connections.sum { it.neuron.downstream_params_tally }
+    end
+
+    # Sum of activations + 1.  It's a component of the sensitivity measure nju.
+    # See [wiki](https://github.com/carlosjhr64/neuronet/wiki)
+    def mju = 1 + connections.sum { it.neuron.activation }
+
+    # Sensitivity measure nju:
+    #     𝒆 ~ 𝜀𝝁 + 𝑾 𝓑𝒂'𝒆'
+    #     𝝂 ≜ 𝒆/𝜀
+    #     𝝂 ~ 𝝁 + 𝑾 𝓑𝒂'𝝂'
+    # See the [wiki](https://github.com/carlosjhr64/neuronet/wiki)
+    # See also test/tc_epsilon:
+    #     https://github.com/carlosjhr64/neuronet/blob/master/test/tc_epsilon
+    # rubocop: disable Metrics
+    def nju
+      return 0 if connections.empty?
+
+      mju + connections.sum do |connection|
+        n = connection.neuron
+        next 0.0 if (nju = n.nju).zero? || (a = n.activation).zero? || a >= 1.0
+
+        connection.weight * a * (1.0 - a) * nju
+      end
+    end
+    # rubocop: enable Metrics
+  end
+end

data/lib/neuronet/noisy_backpropagate.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Noisy Backpropagate
+  module NoisyBackpropagate
+    # rubocop: disable Metrics, Style
+    def backpropagate(error)
+      bmax = Config.bias_clamp
+      b = bias + (error * (rand + rand))
+      self.bias = b.abs > bmax ? (b.positive? ? bmax : -bmax) : b
+
+      wmax = Config.weight_clamp
+      connections.each do |c|
+        n = c.neuron
+        w = c.weight + (n.activation * error * (rand + rand))
+        c.weight = w.abs > wmax ? (w.positive? ? wmax : -wmax) : w
+        n.backpropagate(error)
+      end
+    end
+    # rubocop: enable Metrics, Style
+  end
+end

data/lib/neuronet/noisy_middle_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyMiddleNeuron < MiddleNeuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/noisy_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyNeuron < Neuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/noisy_output_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyOutputNeuron < OutputNeuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/output_layer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Output Layer
+  class OutputLayer
+    include LayerPresets
+    include Arrayable
+
+    def initialize(length, output_neuron: OutputNeuron)
+      @layer = Array.new(length) { output_neuron.new }
+    end
+
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
+      end
+    end
+
+    def values
+      @layer.map(&:value)
+    end
+
+    def to_a = @layer
+  end
+end

data/lib/neuronet/output_neuron.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Output Neuron
+  class OutputNeuron
+    include NeuronStats
+    include Backpropagate
+
+    def initialize
+      @bias        = 0.0
+      @connections = []
+    end
+
+    attr_accessor :bias
+    attr_reader   :connections
+
+    def activation = nil
+
+    def connect(neuron, weight = 0.0)
+      @connections << Connection.new(neuron, weight)
+    end
+
+    def value
+      @bias + @connections.sum(&:value)
+    end
+  end
+end

data/lib/neuronet/perceptron.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Perceptron
+  class Perceptron
+    include NetworkStats
+    include Exportable
+    include Trainable
+    include Arrayable
+
+    def initialize(input_size, output_size,
+                   input_neuron: InputNeuron, output_neuron: OutputNeuron)
+      @input_layer = InputLayer.new(input_size, input_neuron:)
+      @output_layer = OutputLayer.new(output_size, output_neuron:)
+      @output_layer.connect(@input_layer)
+    end
+
+    attr_reader :input_layer, :output_layer
+
+    def set(values)
+      @input_layer.set(values)
+    end
+
+    def values
+      @output_layer.values
+    end
+
+    def *(other)
+      set(other)
+      values
+    end
+
+    def to_a = [@input_layer, @output_layer]
+  end
+end