neuronet 6.1.0 → 7.0.230416

data/lib/neuronet/connection.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Neuronet module / Connection class
+module Neuronet
+  # Connections between neurons are there own separate objects. In Neuronet, a
+  # neuron contains it's bias, and a list of it's connections. Each connection
+  # contains it's weight (strength) and connected neuron.
+  class Connection
+    attr_accessor :neuron, :weight
+
+    # Connection#initialize takes a neuron and a weight with a default of 0.0.
+    def initialize(neuron = Neuron.new, weight: 0.0)
+      @neuron = neuron
+      @weight = weight
+    end
+
+    # The connection's mu is the activation of the connected neuron.
+    def mu = @neuron.activation
+    alias activation mu
+
+    # The connection's mju is 𝑾𝓑𝒂'.
+    def mju = @weight * @neuron.derivative
+
+    # The connection kappa is a component of the neuron's sum kappa:
+    #   𝜿 := 𝑾 𝝀'
+    def kappa = @weight * @neuron.lamda
+
+    # The weighted activation of the connected neuron.
+    def weighted_activation = @neuron.activation * @weight
+
+    # Consistent with #update
+    alias partial weighted_activation
+
+    # Connection#update returns the updated activation of a connection, which is
+    # the weighted updated activation of the neuron it's connected to:
+    #   weight * neuron.update
+    # This method is the one to use whenever the value of the inputs are changed
+    # (or right after training).  Otherwise, both update and value should give
+    # the same result.  When back calculation are not needed, use
+    # Connection#weighted_activation instead.
+    def update = @neuron.update * @weight
+
+    # Connection#backpropagate modifies the connection's weight in proportion to
+    # the error given and passes that error to its connected neuron via the
+    # neuron's backpropagate method.
+    def backpropagate(error)
+      @weight += @neuron.activation * Neuronet.noise[error]
+      if @weight.abs > Neuronet.maxw
+        @weight = @weight.positive? ? Neuronet.maxw : -Neuronet.maxw
+      end
+      @neuron.backpropagate(error)
+      self
+    end
+    # On how to reduce the error, the above makes it obvious how to interpret
+    # the equipartition of errors among the connections.  Backpropagation is
+    # symmetric to forward propagation of errors. The error variable is the
+    # reduced error, 𝛆(see the wiki notes).
+
+    # A connection inspects itself as "weight*label:...".
+    def inspect = "#{Neuronet.format % @weight}*#{@neuron.inspect}"
+
+    # A connection puts itself as "weight*label".
+    def to_s = "#{Neuronet.format % @weight}*#{@neuron}"
+  end
+end

data/lib/neuronet/constants.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Neuronet module / Constants
+module Neuronet
+  # Neuronet allows one to set the format to use for displaying float values,
+  # mostly used in the inspect methods.
+  # [Docs](https://docs.ruby-lang.org/en/master/format_specifications_rdoc.html)
+  FORMAT = '%.13g'
+
+  # An artificial neural network uses a squash function to determine the
+  # activation value of a neuron.  The squash function for Neuronet is the
+  # [Sigmoid function](http://en.wikipedia.org/wiki/Sigmoid_function) which sets
+  # the neuron's activation value between 0.0 and 1.0.  This activation value is
+  # often thought of on/off or true/false.  For classification problems,
+  # activation values near one are considered true while activation values near
+  # 0.0 are considered false.  In Neuronet I make a distinction between the
+  # neuron's activation value and it's representation to the problem.  This
+  # attribute, activation, need never appear in an implementation of Neuronet,
+  # but it is mapped back to it's unsquashed value every time the implementation
+  # asks for the neuron's value.  One should scale the problem with most data
+  # points between -1 and 1, extremes under 2s, and no outbounds above 3s.
+  # Standard deviations from the mean is probably a good way to figure the scale
+  # of the problem.
+  SQUASH     = ->(unsquashed) { 1.0 / (1.0 + Math.exp(-unsquashed)) }
+  UNSQUASH   = ->(squashed) { Math.log(squashed / (1.0 - squashed)) }
+  DERIVATIVE = ->(squash) { squash * (1.0 - squash) }
+
+  # I'll want to have a neuron roughly mirror another later.   Let [v] be the
+  # squash of v.  Consider:
+  #   v = b + w*[v]
+  # There is no constant b and w that will satisfy the above equation for all v.
+  # But one can satisfy the equation for v in {-1, 0, 1}.  Find b and w such
+  # that:
+  #   A: 0 = b + w*[0]
+  #   B: 1 = b + w*[1]
+  #   C: -1 = b + w*[-1]
+  # Use A and B to solve for b and w:
+  #   A: 0 = b + w*[0]
+  #      b = -w*[0]
+  #   B: 1 = b + w*[1]
+  #      1 = -w*[0] + w*[1]
+  #      1 = w*(-[0] + [1])
+  #      w = 1/([1] - [0])
+  #      b = -[0]/([1] - [0])
+  # Verify A, B, and C:
+  #   A: 0 = b + w*[0]
+  #      0 = -[0]/([1] - [0]) + [0]/([1] - [0])
+  #      0 = 0 # OK
+  #   B: 1 = b + w*[1]
+  #      1 = -[0]/([1] - [0]) + [1]/([1] - [0])
+  #      1 = ([1] - [0])/([1] - [0])
+  #      1 = 1 # OK
+  # Using the squash function identity, [v] = 1 - [-v]:
+  #   C: -1 = b + w*[-1]
+  #      -1 = -[0]/([1] - [0]) + [-1]/([1] - [0])
+  #      -1 = ([-1] - [0])/([1] - [0])
+  #      [0] - [1] = [-1] - [0]
+  #      [0] - [1] = 1 - [1] - [0] # Identity substitution.
+  #      [0] = 1 - [0] # OK, by identity(0=-0).
+  # Evaluate given that [0] = 0.5:
+  #      b = -[0]/([1] - [0])
+  #      b = [0]/([0] - [1])
+  #      b = 0.5/(0.5 - [1])
+  #      w = 1/([1] - [0])
+  #      w = 1/([1] - 0.5)
+  #      w = -2 * 0.5/(0.5 - [1])
+  #      w = -2 * b
+  BZERO = 0.5 / (0.5 - SQUASH[1.0])
+  WONE  = -2.0 * BZERO
+
+  # Although the implementation is free to set all parameters for each neuron,
+  # Neuronet by default creates zeroed neurons.  Association between inputs and
+  # outputs are trained, and neurons differentiate from each other randomly.
+  # Differentiation among neurons is achieved by noise in the back-propagation
+  # of errors.  This noise is provided by rand + rand.  I chose rand + rand to
+  # give the noise an average value of one and a bell shape distribution.
+  NOISE = ->(error) { error * (rand + rand) }
+
+  # One may choose not to have noise.
+  NO_NOISE = ->(error) { error }
+
+  # To keep components bounded, Neuronet limits the weights, biases, and values.
+  # Note that on a 64-bit machine SQUASH[37] rounds to 1.0, and
+  # SQUASH[9] is 0.99987...
+  MAXW = 9.0  # Maximum weight
+  MAXB = 18.0 # Maximum bias
+  MAXV = 36.0 # Maximum value
+
+  # Mu learning factor.
+  LEARNING = 1.0
+
+  # The above constants are the defaults for Neuronet.  They are set below in
+  # accessable module attributes.  The user may change these to suit their
+  # needs.
+  class << self
+    attr_accessor :format, :squash, :unsquash, :derivative, :bzero, :wone,
+                  :noise, :maxw, :maxb, :maxv, :learning
+  end
+  self.squash     = SQUASH
+  self.unsquash   = UNSQUASH
+  self.derivative = DERIVATIVE
+  self.bzero      = BZERO
+  self.wone       = WONE
+  self.noise      = NOISE
+  self.format     = FORMAT
+  self.maxw       = MAXW
+  self.maxb       = MAXB
+  self.maxv       = MAXV
+  self.learning   = LEARNING
+end

data/lib/neuronet/feed_forward.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# Neuronet module / FeedForward class
+module Neuronet
+  # A Feed Forward Network
+  class FeedForward < Array
+    # Example:
+    #   ff = Neuronet::FeedForward.new([2, 3, 1])
+    def initialize(layers)
+      length = layers.length
+      raise 'Need at least 2 layers' if length < 2
+
+      super(length) { Layer.new(layers[_1]) }
+      1.upto(length - 1) { self[_1].connect(self[_1 - 1]) }
+    end
+
+    # Set the input layer.
+    def set(input)
+      first.set(input)
+      self
+    end
+
+    def input = first.values
+
+    # Update the network.
+    def update
+      # update up the layers
+      1.upto(length - 1) { self[_1].partial }
+      self
+    end
+
+    def output = last.values
+
+    # Consider:
+    #   m = Neuronet::FeedForward.new(layers)
+    # Want:
+    #   output = m * input
+    def *(other)
+      set(other)
+      update
+      last.values
+    end
+
+    # 𝝁 + 𝜧 𝝁' + 𝜧 𝜧'𝝁" + 𝜧 𝜧'𝜧"𝝁"' + ...
+    # |𝜧| ~ |𝑾||𝓑𝒂|
+    # |∑𝑾| ~ √𝑁
+    # |𝓑𝒂| ~ ¼
+    # |𝝁| ~ 1+∑|𝒂'| ~ 1+½𝑁
+    def expected_mju!
+      sum = 0.0
+      mju = 1.0
+      reverse[1..].each do |layer|
+        n = layer.length
+        sum += mju * (1.0 + (0.5 * n))
+        mju *= 0.25 * Math.sqrt(layer.length)
+      end
+      @expected_mju = Neuronet.learning * sum
+    end
+
+    def expected_mju
+      @expected_mju || expected_mju!
+    end
+
+    def average_mju
+      last.average_mju
+    end
+
+    def train(target, mju = expected_mju)
+      last.train(target, mju)
+      self
+    end
+
+    def pair(input, target, mju = expected_mju)
+      set(input).update.train(target, mju)
+    end
+
+    def pairs(pairs, mju = expected_mju)
+      pairs.shuffle.each { |input, target| pair(input, target, mju) }
+      return self unless block_given?
+
+      pairs.shuffle.each { |i, t| pair(i, t, mju) } while yield
+      self
+    end
+
+    def inspect = map(&:inspect).join("\n")
+
+    def to_s = map(&:to_s).join("\n")
+  end
+end

data/lib/neuronet/gaussian.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Neuronet module
+module Neuronet
+  # "Normal Distribution"
+  # Gaussian sub-classes Scale and is used exactly the same way. The only
+  # changes are that it calculates the arithmetic mean (average) for center and
+  # the standard deviation for spread.
+  class Gaussian < Scale
+    def set(inputs)
+      @center ||= inputs.sum.to_f / inputs.length
+      unless @spread
+        sum2 = inputs.map { @center - _1 }.sum { _1 * _1 }.to_f
+        @spread = Math.sqrt(sum2 / (inputs.length - 1.0))
+      end
+      self
+    end
+  end
+end

data/lib/neuronet/layer.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# Neuronet module
+module Neuronet
+  # Layer is an array of neurons.
+  class Layer < Array
+    # Length is the number of neurons in the layer.
+    def initialize(length)
+      super(length) { Neuron.new }
+    end
+
+    # This is where one enters the "real world" inputs.
+    def set(inputs)
+      0.upto(length - 1) { self[_1].value = inputs[_1] || 0.0 }
+      self
+    end
+
+    # Returns the real world values: [value, ...]
+    def values
+      map(&:value)
+    end
+
+    # Allows one to fully connect layers.
+    def connect(layer = Layer.new(length), weights: [])
+      # creates the neuron matrix...
+      each_with_index do |neuron, i|
+        weight = weights[i] || 0.0
+        layer.each { neuron.connect(_1, weight:) }
+      end
+      # The layer is returned for chaining.
+      layer
+    end
+
+    # Set layer to mirror input:
+    #   bias   = BZERO.
+    #   weight = WONE
+    # Input should be the same size as the layer.  One can set sign to -1 to
+    # anti-mirror.  One can set sign to other than |1| to scale.
+    def mirror(sign = 1)
+      each_with_index do |neuron, index|
+        neuron.bias = sign * Neuronet.bzero
+        neuron.connections[index].weight = sign * Neuronet.wone
+      end
+    end
+
+    # Doubles up the input mirroring and anti-mirroring it.  The layer should be
+    # twice the size of the input.
+    def antithesis
+      sign = 1
+      each_with_index do |n, i|
+        n.connections[i / 2].weight = sign * Neuronet.wone
+        n.bias = sign * Neuronet.bzero
+        sign = -sign
+      end
+    end
+
+    # Sums two corresponding input neurons above each neuron in the layer.
+    # Input should be twice the size of the layer.
+    def synthesis
+      semi = Neuronet.wone / 2
+      each_with_index do |n, i|
+        j = i * 2
+        c = n.connections
+        n.bias = Neuronet.bzero
+        c[j].weight = semi
+        c[j + 1].weight = semi
+      end
+    end
+
+    # Set layer to average input.
+    def average(sign = 1)
+      bias = sign * Neuronet.bzero
+      each do |n|
+        n.bias = bias
+        weight = sign * Neuronet.wone / n.connections.length
+        n.connections.each { _1.weight = weight }
+      end
+    end
+
+    # updates layer with current values of the previous layer
+    def partial
+      each(&:partial)
+    end
+
+    def average_mju
+      Neuronet.learning * sum { Neuron.mju(_1) } / length
+    end
+
+    # Takes the real world target for each neuron in this layer and
+    # backpropagates the error to each neuron.
+    def train(target, mju = nil)
+      0.upto(length - 1) do |index|
+        neuron = self[index]
+        error = (target[index] - neuron.value) /
+                (mju || (Neuronet.learning * Neuron.mju(neuron)))
+        neuron.backpropagate(error)
+      end
+      self
+    end
+
+    # Layer inspects as "label:value,..."
+    def inspect
+      map(&:inspect).join(',')
+    end
+
+    # Layer puts as "label,..."
+    def to_s
+      map(&:to_s).join(',')
+    end
+  end
+end

data/lib/neuronet/log_normal.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Neuronet module
+module Neuronet
+  # "Log-Normal Distribution"
+  # LogNormal sub-classes Gaussian to transform the values to a logarithmic
+  # scale.
+  class LogNormal < Gaussian
+    def set(inputs)
+      super(inputs.map { |value| Math.log(value) })
+    end
+
+    def mapped(inputs)
+      super(inputs.map { |value| Math.log(value) })
+    end
+
+    def unmapped(outputs)
+      super(outputs).map { |value| Math.exp(value) }
+    end
+  end
+end

data/lib/neuronet/neuron.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+# Neuronet module / Neuron class
+module Neuronet
+  # A Neuron is capable of creating connections to other neurons.  The
+  # connections attribute is a list of the neuron's connections to other
+  # neurons.  A neuron's bias is it's kicker (or deduction) to it's activation
+  # value, a sum of its connections values.
+  class Neuron
+    # For bookkeeping, each Neuron is given a label, starting with 'a' by
+    # default.
+    class << self; attr_accessor :label; end
+    Neuron.label = 'a'
+
+    attr_reader :label, :activation, :connections
+    attr_accessor :bias
+
+    # The neuron's mu is the sum of the connections' mu(activation), plus one
+    # for the bias:
+    #   𝛍 := 1+∑𝐚'
+    def mu
+      return 0.0 if @connections.empty?
+
+      1 + @connections.sum(&:mu)
+    end
+
+    # Reference the library's wiki:
+    #   𝒆ₕ ~ 𝜀(𝝁ₕ + 𝜧ₕⁱ𝝁ᵢ + 𝜧ₕⁱ𝜧ᵢʲ𝝁ⱼ + 𝜧ₕⁱ𝜧ᵢʲ𝜧ⱼᵏ𝝁ₖ + ...)
+    # 𝜧ₕⁱ𝝁ᵢ is:
+    #   neuron.mju{ |connected_neuron| connected_neuron.mu }
+    # 𝜧ₕⁱ𝜧ᵢʲ𝝁ⱼ is:
+    #   nh.mju{ |ni| ni.mju{ |nj| nj.mu }}
+    def mju(&block)
+      @connections.sum { _1.mju * block[_1.neuron] }
+    end
+
+    # Full recursive implementation of mju:
+    def self.mju(neuron)
+      return 0.0 if neuron.connections.empty?
+
+      neuron.mu + neuron.mju { |connected_neuron| Neuron.mju(connected_neuron) }
+    end
+
+    # 𝓓𝒗⌈𝒗 = (1-⌈𝒗)⌈𝒗 = (1-𝒂)𝒂 = 𝓑𝒂
+    def derivative = Neuronet.derivative[@activation]
+
+    # 𝝀 = 𝓑𝒂𝛍
+    def lamda = derivative * mu
+
+    # 𝜿 := 𝜧 𝝁' = 𝑾 𝓑𝒂'𝝁' = 𝑾 𝝀'
+    # def kappa = mju(&:mu)
+    def kappa = @connections.sum(&:kappa)
+
+    # 𝜾 := 𝜧 𝜧' 𝝁" = 𝜧 𝜿'
+    def iota = mju(&:kappa)
+
+    # One can explicitly set the neuron's value, typically used to set the input
+    # neurons.  The given "real world" value is squashed into the neuron's
+    # activation value.
+    def value=(value)
+      # If value is out of bounds, set it to the bound.
+      if value.abs > Neuronet.maxv
+        value = value.positive? ? Neuronet.maxv : -Neuronet.maxv
+      end
+      @activation = Neuronet.squash[value]
+    end
+
+    # The "real world" value of the neuron is the unsquashed activation value.
+    def value = Neuronet.unsquash[@activation]
+
+    # The initialize method sets the neuron's value, bias and connections.
+    def initialize(value = 0.0, bias: 0.0, connections: [])
+      self.value   = value
+      @connections = connections
+      @bias        = bias
+      @label       = Neuron.label
+      Neuron.label = Neuron.label.next
+    end
+
+    # Updates the activation with the current value of bias and updated values
+    # of connections.
+    def update
+      return @activation if @connections.empty?
+
+      self.value = @bias + @connections.sum(&:update)
+      @activation
+    end
+
+    # For when connections are already updated, Neuron#partial updates the
+    # activation with the current values of bias and connections.  It is not
+    # always necessary to burrow all the way down to the terminal input neuron
+    # to update the current neuron if it's connected neurons have all been
+    # updated.  The implementation should set it's algorithm to use partial
+    # instead of update as update will most likely needlessly update previously
+    # updated neurons.
+    def partial
+      return @activation if @connections.empty?
+
+      self.value = @bias + @connections.sum(&:partial)
+      @activation
+    end
+
+    # The backpropagate method modifies the neuron's bias in proportion to the
+    # given error and passes on this error to each of its connection's
+    # backpropagate method.  While updates flows from input to output, back-
+    # propagation of errors flows from output to input.
+    def backpropagate(error)
+      return self if @connections.empty?
+
+      @bias += Neuronet.noise[error]
+      if @bias.abs > Neuronet.maxb
+        @bias = @bias.positive? ? Neuronet.maxb : -Neuronet.maxb
+      end
+      @connections.each { |connection| connection.backpropagate(error) }
+      self
+    end
+
+    # Connects the neuron to another neuron.  The default weight=0 means there
+    # is no initial association.  The connect method is how the implementation
+    # adds a connection, the way to connect a neuron to another.  To connect
+    # "output" to "input", for example, it is:
+    #	  input = Neuronet::Neuron.new
+    #	  output = Neuronet::Neuron.new
+    #	  output.connect(input)
+    # Think "output" connects to "input".
+    def connect(neuron = Neuron.new, weight: 0.0)
+      @connections.push(Connection.new(neuron, weight:))
+      # Note that we're returning the connected neuron:
+      neuron
+    end
+
+    # Tacks on to neuron's inspect method to show the neuron's bias and
+    # connections.
+    def inspect
+      fmt = Neuronet.format
+      if @connections.empty?
+        "#{@label}:#{fmt % value}"
+      else
+        "#{@label}:#{fmt % value}|#{[(fmt % @bias), *@connections].join('+')}"
+      end
+    end
+
+    # A neuron plainly puts itself as it's label.
+    def to_s = @label
+  end
+end

data/lib/neuronet/scale.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Neuronet module
+module Neuronet
+  # Neuronet::Scale is a class to help scale problems to fit within a network's
+  # "field of view". Given a list of values, it finds the minimum and maximum
+  # values and establishes a mapping to a scaled set of numbers between minus
+  # one and one (-1,1).
+  class Scale
+    attr_accessor :spread, :center
+
+    # If the value of center is provided, then
+    # that value will be used instead of
+    # calculating it from the values passed to method #set.
+    # Likewise, if spread is provided, that value of spread will be used.
+    def initialize(factor: 1.0, center: nil, spread: nil)
+      @factor = factor
+      @center = center
+      @spread = spread
+    end
+
+    def set(inputs)
+      min, max = inputs.minmax
+      @center ||= (max + min) / 2.0
+      @spread ||= (max - min) / 2.0
+      self
+    end
+
+    def reset(inputs)
+      @center = @spread = nil
+      set(inputs)
+    end
+
+    def mapped(inputs)
+      factor = 1.0 / (@factor * @spread)
+      inputs.map { |value| factor * (value - @center) }
+    end
+    alias mapped_input mapped
+    alias mapped_output mapped
+
+    # Note that it could also unmap inputs, but
+    # outputs is typically what's being transformed back.
+    def unmapped(outputs)
+      factor = @factor * @spread
+      outputs.map { |value| (factor * value) + @center }
+    end
+    alias unmapped_input unmapped
+    alias unmapped_output unmapped
+  end
+end

data/lib/neuronet/scaled_network.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Neuronet module
+module Neuronet
+  # ScaledNetwork is a subclass of FeedForwardNetwork.
+  # It automatically scales the problem given to it
+  # by using a Scale type instance set in @distribution.
+  # The attribute, @distribution, is set to Neuronet::Gaussian.new by default,
+  # but one can change this to Scale, LogNormal, or one's own custom mapper.
+  class ScaledNetwork < FeedForward
+    attr_accessor :distribution, :reset
+
+    def initialize(layers, distribution: Gaussian.new, reset: false)
+      super(layers)
+      @distribution = distribution
+      @reset        = reset
+    end
+
+    # ScaledNetwork set works just like FeedForwardNetwork's set method,
+    # but calls @distribution.set(values) first if @reset is true.
+    # Sometimes you'll want to set the distribution with the entire data set,
+    # and then there will be times you'll want to reset the distribution
+    # with each input.
+    def set(input)
+      @distribution.reset(input) if @reset
+      super(@distribution.mapped_input(input))
+    end
+
+    def input
+      @distribution.unmapped_input(super)
+    end
+
+    def output
+      @distribution.unmapped_output(super)
+    end
+
+    def *(_other)
+      @distribution.unmapped_output(super)
+    end
+
+    def train(target, mju = expected_mju)
+      super(@distribution.mapped_output(target), mju)
+    end
+
+    def inspect
+      distribution = @distribution.class.to_s.split(':').last
+      "#distribution:#{distribution} #reset:#{@reset}\n" + super
+    end
+  end
+end