neuronet 6.1.0 → 8.0.251113

data/lib/neuronet/arrayable.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Arrayable avoids explicit `to_a` calls for common Array methods.
+  module Arrayable
+    def each(&blk) = to_a.each { blk[it] }
+    def each_with_index(&blk) = to_a.each_with_index { |n, i| blk[n, i] }
+    def [](index) = to_a[index]
+    def map(&) = to_a.map(&)
+    def size = to_a.size
+    def reverse = to_a.reverse
+  end
+end

data/lib/neuronet/backpropagate.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Backpropagate provides simple, clamp-limited weight/bias updates.
+  module Backpropagate
+    # Back-propagates errors, updating bias and connection weights.
+    # Clamps updates to [-max, +max].
+    # Recursively calls on connected neurons.
+    # rubocop: disable Metrics, Style
+    def backpropagate(error)
+      bmax = Config.bias_clamp
+      b = bias + error
+      self.bias = b.abs > bmax ? (b.positive? ? bmax : -bmax) : b
+
+      wmax = Config.weight_clamp
+      connections.each do |c|
+        n = c.neuron
+        w = c.weight + (n.activation * error)
+        c.weight = w.abs > wmax ? (w.positive? ? wmax : -wmax) : w
+        n.backpropagate(error)
+      end
+    end
+    # rubocop: enable Metrics, Style
+  end
+end

data/lib/neuronet/config.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Maximum values for biases and weights
+  module Config
+    class << self; attr_accessor :bias_clamp, :weight_clamp; end
+    self.bias_clamp   = 18.0
+    self.weight_clamp = 9.0
+  end
+end

data/lib/neuronet/connection.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Connection is a lightweight struct for weighted neuron links.
+  Connection = Struct.new('Connection', :neuron, :weight) do
+    # Weighted activation value
+    def value = neuron.activation * weight
+  end
+end

data/lib/neuronet/deep.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Feed Forward
+  class Deep
+    include NetworkStats
+    include Exportable
+    include Trainable
+    include Arrayable
+
+    # rubocop: disable Metrics
+    def initialize(*sizes, input_neuron: InputNeuron,
+                   middle_neuron: MiddleNeuron,
+                   output_neuron: OutputNeuron)
+      length = sizes.length
+      raise 'Need at least 3 layers' if length < 3
+
+      @input_layer = InputLayer.new(sizes.shift, input_neuron:)
+      @output_layer = OutputLayer.new(sizes.pop, output_neuron:)
+      @hidden_layers = sizes.map { MiddleLayer.new(it, middle_neuron:) }
+      previous = @input_layer
+      @hidden_layers.each do |layer|
+        layer.connect(previous)
+        previous = layer
+      end
+      @output_layer.connect(previous)
+    end
+    # rubocop: enable Metrics
+
+    attr_reader :input_layer, :hidden_layers, :output_layer
+
+    def set(values)
+      @input_layer.set(values)
+    end
+
+    def update
+      @hidden_layers.each(&:update)
+    end
+
+    def values
+      @output_layer.values
+    end
+
+    def *(other)
+      set(other)
+      update
+      values
+    end
+
+    def to_a = [@input_layer, *@hidden_layers, @output_layer]
+  end
+end

data/lib/neuronet/exportable.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Exportable serializes network biases and weights only.
+  # Human-readable, compact, excludes activations.
+  module Exportable
+    # Writes serialized network to writer(from self).
+    # rubocop: disable Metrics
+    def export(writer)
+      sizes = map(&:size)
+      writer.puts "# #{self.class}"
+      # The first "float" here is the number of layers in the FFN...
+      # Just to be consistent:
+      writer.puts "#{sizes.size.to_f} #{sizes.join(' ')}"
+      each_with_index do |layer, i|
+        next if i.zero? # skip input layer
+
+        layer.each_with_index do |neuron, j|
+          writer.puts "# neuron = FFN[#{i}, #{j}]"
+          writer.puts "#{neuron.bias} #{i} #{j}"
+          neuron.connections.each_with_index do |connection, k|
+            writer.puts "#{connection.weight} #{i} #{j} #{k}"
+          end
+        end
+      end
+    end
+    # rubocop: enable Metrics
+
+    def export_to_file(filename) = File.open(filename, 'w') { export it }
+    def import_from_file(filename) = File.open(filename, 'r') { import it }
+
+    # Reads and validates serialized network from reader to set self.
+    # rubocop: disable Metrics
+    def import(reader)
+      gets_data = lambda do |reader|
+        return nil unless (line = reader.gets)
+
+        line = reader.gets while line.start_with?('#')
+        fs, *is = line.strip.split
+        [fs.to_f, *is.map(&:to_i)]
+      end
+
+      size, *sizes = gets_data[reader]
+      raise 'Size/Sizes mismatch' unless size == sizes.size
+      raise 'Sizes mismatch' unless sizes == map(&:size)
+
+      each_with_index do |layer, i|
+        next if i.zero? # skip input layer
+
+        layer.each_with_index do |neuron, j|
+          bias, *indeces = gets_data[reader]
+          raise "bad bias index: #{indeces}" unless indeces == [i, j]
+
+          neuron.bias = bias
+          neuron.connections.each_with_index do |connection, k|
+            weight, *indeces = gets_data[reader]
+            raise "bad weight index: #{indeces}" unless indeces == [i, j, k]
+
+            connection.weight = weight
+          end
+        end
+      end
+      raise 'Expected end of file.' unless gets_data[reader].nil?
+    end
+    # rubocop: enable Metrics
+  end
+end

data/lib/neuronet/feed_forward.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # FeedForward is a fully connected neural network with >= 3 layers.
+  class FeedForward
+    # [NetwordStats](network_stats.rb)
+    include NetworkStats
+    # [Exportable](exportable.rb)
+    include Exportable
+    # [Trainable](trainable.rb)
+    include Trainable
+    # [Arrayble](arrayable.rb)
+    include Arrayable
+
+    # Example:
+    #     ff = Neuronet::FeedForward.new(4, 8, 4)
+    def initialize(*sizes, full_neuron: Neuron)
+      length = sizes.length
+      raise 'Need at least 3 layers' if length < 3
+
+      @layers = Array.new(length) { Layer.new(sizes[it], full_neuron:) }
+      1.upto(length - 1) { @layers[it].connect(@layers[it - 1]) }
+      @input_layer = @layers[0]
+      @output_layer = @layers[-1]
+      @hidden_layers = @layers[1...-1]
+    end
+
+    attr_reader :input_layer, :hidden_layers, :output_layer
+
+    # Sets the input values
+    def set(values)
+      @input_layer.set(values)
+    end
+
+    # Updates hidden layers (input assumed set).
+    def update
+      @hidden_layers.each(&:update)
+    end
+
+    # Gets output
+    def values
+      @output_layer.values
+    end
+
+    # Forward pass: set input, update, return output.
+    def *(other)
+      set(other)
+      update
+      values
+    end
+
+    def to_a = @layers
+  end
+end

data/lib/neuronet/input_layer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Input Layer
+  class InputLayer
+    include Arrayable
+
+    def initialize(length, input_neuron: InputNeuron)
+      @layer = Array.new(length) { input_neuron.new }
+      @endex = length - 1
+    end
+
+    def set(values)
+      0.upto(@endex) { @layer[it].set values[it] }
+    end
+
+    def to_a = @layer
+  end
+end

data/lib/neuronet/input_neuron.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Input Neuron
+  class InputNeuron
+    include NeuronStats
+    include Squash
+
+    EMPTY = [].freeze
+
+    def initialize
+      @activation = 0.5
+    end
+
+    attr_reader :activation
+
+    def bias = nil
+    def connections = EMPTY
+    def value = nil
+
+    def set(value)
+      @activation = squash(value)
+    end
+
+    def backpropagate(_) = nil
+  end
+end

data/lib/neuronet/layer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Layer is a collection of neurons with array-like behavior.
+  class Layer
+    # [LayerPresets](layer_presets.rb)
+    include LayerPresets
+    # [Arrayable](arrayable.rb)
+    include Arrayable
+
+    # Creates layer with `length` number of neurons.
+    def initialize(length, full_neuron: Neuron)
+      @layer = Array.new(length) { full_neuron.new }
+      @endex = length - 1
+    end
+
+    # Set each neuron's activation from values array.
+    # Allows the layer to be used as an input layer.
+    def set(values)
+      0.upto(@endex) { @layer[it].set values[it] }
+    end
+
+    # For each neuron in the layer, updates the neuron's activation.
+    def update = @layer.each(&:update)
+
+    # Fully connects this layer to another.
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
+      end
+    end
+
+    # Raw pre-squashed values of all neurons in the layer.
+    # Allows the layer to be use as an output layer.
+    def values
+      @layer.map(&:value)
+    end
+
+    def to_a = @layer
+  end
+end

data/lib/neuronet/layer_presets.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # LayerPresets initializes layer weights/biases for interpret-able functions.
+  module LayerPresets
+    BZERO = 0.5 / (0.5 - Squash.squash(1.0))
+    WONE = -2.0 * BZERO
+
+    # Set layer to roughly mirror it's input.
+    # Input should be the same size as the layer.
+    def mirror(sign = 1.0)
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        neuron.connections[index].weight = sign * WONE
+      end
+    end
+
+    # Doubles up the input both mirroring and anti-mirroring it.
+    # The layer should be twice the size of the input.
+    def antithesis
+      sign = 1.0
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        neuron.connections[index / 2].weight = sign * WONE
+        sign = -sign
+      end
+    end
+
+    # Sums two corresponding input neurons above each neuron in the layer.
+    # Input should be twice the size of the layer.
+    def synthesis(sign = 1.0)
+      semi = sign * WONE / 2.0
+      each_with_index do |neuron, index|
+        neuron.bias = sign * BZERO
+        j = index * 2
+        connections = neuron.connections
+        connections[j].weight = semi
+        connections[j + 1].weight = semi
+      end
+    end
+
+    # Set layer to average input.
+    def average(sign = 1.0)
+      bias = sign * BZERO
+      each do |neuron|
+        neuron.bias = bias
+        connections = neuron.connections
+        weight = sign * WONE / connections.size
+        connections.each { it.weight = weight }
+      end
+    end
+  end
+end

data/lib/neuronet/middle_layer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Neuron Layer
+  class MiddleLayer
+    include LayerPresets
+    include Arrayable
+
+    def initialize(length, middle_neuron: MiddleNeuron)
+      @layer = Array.new(length) { middle_neuron.new }
+    end
+
+    def update = @layer.each(&:update)
+
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
+      end
+    end
+
+    def to_a = @layer
+  end
+end

data/lib/neuronet/middle_neuron.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class MiddleNeuron
+    include NeuronStats
+    include Backpropagate
+    include Squash
+
+    def initialize
+      @activation  = 0.5
+      @bias        = 0.0
+      @connections = []
+    end
+
+    attr_accessor :bias
+    attr_reader   :activation, :connections
+
+    def connect(neuron, weight = 0.0)
+      @connections << Connection.new(neuron, weight)
+    end
+
+    def value
+      @bias + @connections.sum(&:value)
+    end
+
+    def update
+      @activation = squash(value)
+    end
+  end
+end

data/lib/neuronet/mlp.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Multi Layer Perceptron(3 layers)
+  class MLP
+    include NetworkStats
+    include Exportable
+    include Trainable
+    include Arrayable
+
+    # rubocop: disable Metrics
+    def initialize(input_size, middle_size, output_size,
+                   input_neuron: InputNeuron,
+                   middle_neuron: MiddleNeuron,
+                   output_neuron: OutputNeuron)
+      @input_layer = InputLayer.new(input_size, input_neuron:)
+      @middle_layer = MiddleLayer.new(middle_size, middle_neuron:)
+      @output_layer = OutputLayer.new(output_size, output_neuron:)
+      @middle_layer.connect(@input_layer)
+      @output_layer.connect(@middle_layer)
+    end
+    # rubocop: enable Metrics
+
+    attr_reader :input_layer, :middle_layer, :output_layer
+
+    def set(values)
+      @input_layer.set(values)
+    end
+
+    def update
+      @middle_layer.update
+    end
+
+    def values
+      @output_layer.values
+    end
+
+    def *(other)
+      set(other)
+      update
+      values
+    end
+
+    def to_a = [@input_layer, @middle_layer, @output_layer]
+  end
+end

data/lib/neuronet/network_stats.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Network Stats
+  module NetworkStats
+    # See https://github.com/carlosjhr64/neuronet/wiki
+    # |𝝂| = 𝔪 + ¼√𝑁*𝔪' + ¼√𝑁*¼√𝑁'*𝔪" + ...
+    def expected_nju!
+      nju = 0.0
+      mult = 1.0
+      reverse[1..].each do |layer|
+        size = layer.size
+        mju = 1 + (0.5 * size)
+        nju += mult * mju
+        mult *= 0.25 * Math.sqrt(size)
+      end
+      @expected_nju = nju
+    end
+
+    def expected_nju
+      @expected_nju || expected_nju!
+    end
+
+    def njus
+      output_layer.map(&:nju)
+    end
+  end
+end

data/lib/neuronet/neuron.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Neuron represents a single node in a neural network.
+  # It holds @activation, @bias, and incoming @connections.
+  class Neuron
+    # [nju computation](neuron_stats.rb)
+    include NeuronStats
+    # [back-propagation of errors](backpropagate.rb)
+    include Backpropagate
+    # [squash/unsquash methods](squash.rb)
+    include Squash
+
+    # Initializes a neuron with default activation 0.5 and zero bias.
+    def initialize
+      @activation  = 0.5
+      @bias        = 0.0
+      @connections = [] # incoming connections
+    end
+
+    attr_accessor :bias # bias is settable
+    attr_reader   :activation, :connections # activation is read-only
+
+    # Sets activation by applying squash to raw input value.
+    def set(value)
+      @activation = squash(value)
+    end
+
+    # Creates a weighted connection to another neuron.
+    # See [Neuronet::Connection](connection.rb)
+    def connect(neuron, weight = 0.0)
+      @connections << Connection.new(neuron, weight)
+    end
+
+    # Computes(raw output)value: bias + sum of incoming connection values.
+    def value
+      @bias + @connections.sum(&:value)
+    end
+
+    # Updates activation by squashing the current value(see above).
+    def update
+      @activation = squash(value)
+    end
+  end
+end

data/lib/neuronet/neuron_stats.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # NeuronStats provides network analysis methods.
+  module NeuronStats
+    # Returns the total tally of parameters in the downstream
+    # network subgraph from this neuron.
+    # This includes the neuron's bias (1 parameter),
+    # the weights of its incoming connections (one per connection),
+    # and the sum of parameters from all downstream neurons.
+    # Parameters from a shared neuron are counted multiple times
+    # if accessed via multiple pathways,
+    # reflecting the total parameter influence through all paths.
+    # Returns 0 for a neuron with no downstream connections.
+    def downstream_params_tally
+      return 0 if (size = connections.size).zero?
+
+      1 + size + connections.sum { it.neuron.downstream_params_tally }
+    end
+
+    # Sum of activations + 1.  It's a component of the sensitivity measure nju.
+    # See [wiki](https://github.com/carlosjhr64/neuronet/wiki)
+    def mju = 1 + connections.sum { it.neuron.activation }
+
+    # Sensitivity measure nju:
+    #     𝒆 ~ 𝜀𝝁 + 𝑾 𝓑𝒂'𝒆'
+    #     𝝂 ≜ 𝒆/𝜀
+    #     𝝂 ~ 𝝁 + 𝑾 𝓑𝒂'𝝂'
+    # See the [wiki](https://github.com/carlosjhr64/neuronet/wiki)
+    # See also test/tc_epsilon:
+    #     https://github.com/carlosjhr64/neuronet/blob/master/test/tc_epsilon
+    # rubocop: disable Metrics
+    def nju
+      return 0 if connections.empty?
+
+      mju + connections.sum do |connection|
+        n = connection.neuron
+        next 0.0 if (nju = n.nju).zero? || (a = n.activation).zero? || a >= 1.0
+
+        connection.weight * a * (1.0 - a) * nju
+      end
+    end
+    # rubocop: enable Metrics
+  end
+end

data/lib/neuronet/noisy_backpropagate.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Noisy Backpropagate
+  module NoisyBackpropagate
+    # rubocop: disable Metrics, Style
+    def backpropagate(error)
+      bmax = Config.bias_clamp
+      b = bias + (error * (rand + rand))
+      self.bias = b.abs > bmax ? (b.positive? ? bmax : -bmax) : b
+
+      wmax = Config.weight_clamp
+      connections.each do |c|
+        n = c.neuron
+        w = c.weight + (n.activation * error * (rand + rand))
+        c.weight = w.abs > wmax ? (w.positive? ? wmax : -wmax) : w
+        n.backpropagate(error)
+      end
+    end
+    # rubocop: enable Metrics, Style
+  end
+end

data/lib/neuronet/noisy_middle_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyMiddleNeuron < MiddleNeuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/noisy_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyNeuron < Neuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/noisy_output_neuron.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Middle Neuron
+  class NoisyOutputNeuron < OutputNeuron
+    include NoisyBackpropagate
+  end
+end

data/lib/neuronet/output_layer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Neuronet
+  # Output Layer
+  class OutputLayer
+    include LayerPresets
+    include Arrayable
+
+    def initialize(length, output_neuron: OutputNeuron)
+      @layer = Array.new(length) { output_neuron.new }
+    end
+
+    def connect(layer)
+      each do |neuron|
+        layer.each { neuron.connect(it) }
+      end
+    end
+
+    def values
+      @layer.map(&:value)
+    end
+
+    def to_a = @layer
+  end
+end