CooCoo 0.1.0

data/lib/coo-coo/math/abstract_vector.rb

@@ -0,0 +1,121 @@
+module CooCoo
+  module Math
+    class AbstractVector
+      def self.rand(length, range = nil)
+        new(length) do |i|
+          args = [ range ] if range
+          Random.rand(*args)
+        end
+      end
+
+      def self.zeros(length)
+        new(length, 0.0)
+      end
+
+      def self.ones(length)
+        new(length, 1.0)
+      end
+
+      def zero
+        self.class.zeros(size)
+      end
+
+      def max
+        minmax[1]
+      end
+
+      def min
+        minmax[0]
+      end
+
+      def minmax
+        each.minmax
+      end
+
+      def minmax_normalize(use_zeros = false)
+        min, max = minmax
+        delta = (max - min)
+        if use_zeros && delta == 0.0
+          zero
+        else
+          (self - min) / delta
+        end
+      end
+
+      [ :log, :log2, :log10, :sqrt ].each do |op|
+        define_method(op) do
+          self.class[each.collect(&op)]
+        end
+      end
+
+      def slice_2d(src_width, src_height, origin_x, origin_y, width, height, initial = 0.0)
+        samples = height.times.collect do |y|
+          py = origin_y + y
+
+          width.times.collect do |x|
+            px = origin_x + x
+            if px >= 0 && px < src_width
+              i = py * src_width + px
+              if i >= 0 && i < size
+                self[i]
+              else
+                initial
+              end
+            else
+              initial
+            end
+          end
+        end.flatten
+
+        self.class[samples]
+      end
+
+      def set2d!(width, src, src_width, x, y)
+        raise ArgumentError.new("src's size needs to be a multiple of the width") if src.kind_of?(self.class) && src.size % src_width > 0
+        
+        src.each_slice(src_width).with_index do |row, i|
+          index = (y+i) * width + x
+          next if index >= size
+          row.each_with_index do |p, px|
+            break if (x + px) >= width
+            self[index.to_i + px] = p
+          end
+        end
+
+        self
+      end
+
+      def collect_equal?(n)
+        if n.respond_to?(:each)
+          self.class[each.zip(n).collect { |a, b| a == b ? 1.0 : 0.0 }]
+        else
+          self.class[each.collect { |e| e == n ? 1.0 : 0.0 }]
+        end
+      end
+
+      def collect_not_equal?(n)
+        if n.respond_to?(:each)
+          self.class[each.zip(n).collect { |a, b| a != b ? 1.0 : 0.0 }]
+        else
+          self.class[each.collect { |e| e != n ? 1.0 : 0.0 }]
+        end
+      end
+
+      def collect_nan?
+        self.class[each.collect { |e| e.nan? ? 1.0 : 0.0 }]
+      end
+
+      def nan?
+        each.any?(&:nan?)
+      end
+
+      def collect_infinite?
+        self.class[each.collect { |e| e.infinite? ? 1.0 : 0.0 }]
+      end
+
+      def infinite?
+        each.any?(&:infinite?)
+      end
+    end
+  end
+end

data/lib/coo-coo/math/functions.rb

@@ -0,0 +1,39 @@
+module CooCoo
+  module Math
+    class << self
+      def max(a, b)
+        if a
+          if b
+            (a >= b) ? a : b
+          else
+            a
+          end
+        else
+          b
+        end
+      end
+
+      def min(a, b)
+        if a
+          if b
+            (a <= b) ? a : b
+          else
+            a
+          end
+        else
+          b
+        end
+      end
+
+      def clamp(n, min, max)
+        if n < min
+          min
+        elsif n > max
+          max
+        else
+          n
+        end
+      end
+    end
+  end
+end

data/lib/coo-coo/math/interpolation.rb

@@ -0,0 +1,7 @@
+module CooCoo
+  module Math
+    def self.lerp(a, b, t)
+      a * (1.0 - t) + b * t
+    end
+  end
+end

data/lib/coo-coo/network.rb

@@ -0,0 +1,264 @@
+require 'yaml'
+require 'coo-coo/consts'
+require 'coo-coo/debug'
+require 'coo-coo/core_ext'
+require 'coo-coo/math'
+require 'coo-coo/layer'
+require 'coo-coo/enum'
+require 'coo-coo/cost_functions'
+
+module CooCoo
+  class Network
+    attr_reader :age, :activation_function
+    attr_accessor :command, :comments
+    
+    def initialize
+      @layers = Array.new
+      @age = 0
+      @command = [ $0 ] + ARGV
+      yield(self) if block_given?
+    end
+
+    def num_inputs
+      @layers.first.num_inputs
+    end
+
+    def num_outputs
+      @layers.last.size
+    end
+    
+    def num_layers
+      @layers.size
+    end
+
+    def layers
+      @layers
+    end
+
+    def layer_index(layer)
+      @layers.find_index { |l| l.eql?(layer) }
+    end
+    
+    def layer(new_layer)
+      @layers << new_layer
+      self
+    end
+
+    def activation_function
+      unless @activation_function
+        layer = @layers.find { |l| l.activation_function }
+        @activation_function = layer.activation_function
+      end
+            
+      @activation_function
+    end
+
+    def output_activation_function
+      unless @output_activation_function
+        layer = @layers.reverse.find { |l| l.activation_function }
+        @output_activation_function = layer.activation_function
+      end
+
+      @output_activation_function
+    end
+    
+    def prep_input(input)
+      activation_function.prep_input(input)
+    end
+
+    def prep_output_target(target)
+      output_activation_function.prep_output_target(target)
+    end
+
+    def final_output(outputs)
+      outputs.last
+    end
+    
+    def forward(input, hidden_state = nil, flattened = false, processed = false)
+      unless flattened || input.kind_of?(CooCoo::Vector)
+        input = CooCoo::Vector[input.to_a.flatten, num_inputs]
+      end
+
+      hidden_state ||= Hash.new
+
+      output = if processed
+                 input
+               else
+                 prep_input(input)
+               end
+      
+      outputs = @layers.each_with_index.inject([]) do |acc, (layer, i)|
+        #debug("Layer: #{i} #{layer.num_inputs} #{layer.size}")
+        #debug("Input: #{input}")
+        #debug("Weights: #{layer.neurons[0].weights}")
+        output, hidden_state = layer.forward(output, hidden_state)
+        acc << output
+        #debug("Output: #{input}")
+      end
+
+      return outputs, hidden_state
+    end
+
+    def predict(input, hidden_state = nil, flattened = false, processed = false)
+      hidden_state ||= Hash.new
+      outputs, hidden_state = forward(input, hidden_state, flattened, processed)
+      out = final_output(outputs)
+      return out, hidden_state
+    end
+
+    def backprop(inputs, outputs, errors, hidden_state = nil)
+      hidden_state ||= Hash.new
+      d = @layers.reverse_each.each_with_index.inject([]) do |acc, (layer, i)|
+        input = if i < (@layers.size - 1)
+                  outputs[@layers.size - i - 2]
+                else
+                  prep_input(inputs) # TODO condition prep_input
+                end
+        #CooCoo.debug("#{self.class.name}.#{__method__}\t#{i} #{@layers.size - i - 1}\t#{input.size}\t#{outputs.size}")
+        deltas, hidden_state = layer.backprop(input,
+                                              outputs[@layers.size - i - 1],
+                                              errors,
+                                              hidden_state)
+        errors = layer.transfer_error(deltas)
+        acc.unshift(deltas)
+      end
+
+      return Sequence[d], hidden_state
+    end
+
+    def transfer_errors(deltas)
+      @layers.zip(deltas).collect do |layer, delta|
+        layer.transfer_error(delta)
+      end
+    end
+
+    def update_weights!(input, outputs, deltas)
+      adjust_weights!(weight_deltas(input, outputs, deltas))
+      self
+    end
+
+    def adjust_weights!(deltas)
+      @layers.each_with_index do |layer, i|
+        layer.adjust_weights!(deltas[i])
+      end
+
+      @age += 1
+      self
+    end
+
+    def weight_deltas(input, outputs, deltas)
+      d = @layers.each_with_index.collect do |layer, i|
+        inputs = if i != 0
+                   outputs[i - 1]
+                 else
+                   prep_input(input)
+                 end
+        layer.weight_deltas(inputs, deltas[i])
+      end
+
+      d
+    end
+
+    def learn(input, expecting, rate, cost_function = CostFunctions::MeanSquare, hidden_state = nil)
+      hidden_state ||= Hash.new
+      output, hidden_state = forward(input, hidden_state)
+      cost = cost_function.derivative(prep_input(expecting), output.last)
+      deltas, hidden_state = backprop(input, output, cost, hidden_state)
+      update_weights!(input, output, deltas * rate)
+      return self, hidden_state
+    rescue
+      CooCoo.debug("Network#learn caught #{$!}", input, expecting)
+      raise
+    end
+
+    def save(path)
+      File.write_to(path) do |f|
+        f.write(to_hash.to_yaml)
+      end
+    end
+
+    def load!(path)
+      yaml = YAML.load(File.read(path))
+      raise RuntimeError.new("Invalid YAML definition in #{path}") if yaml.nil?
+        
+      update_from_hash!(yaml)
+
+      self
+    end
+
+    def update_from_hash!(h)
+      @layers = Array.new
+      
+      h[:layers].each do |layer_hash|
+        @layers << CooCoo::LayerFactory.from_hash(layer_hash, self)
+      end
+
+      @age = h.fetch(:age, 0)
+      @command = h.fetch(:command, nil)
+      @comments = h.fetch(:comments) { Array.new }
+
+      self
+    end
+
+    def to_hash
+      { age: @age,
+        command: @command,
+        comments: @comments,
+        layers: @layers.collect { |l| l.to_hash(self) }
+      }
+    end
+
+    class << self
+      def from_a(layers)
+        self.new().update_from_a!(layers)
+      end
+
+      def from_hash(h)
+        self.new.update_from_hash!(h)
+      end
+
+      def load(path)
+        self.new().load!(path)
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  SIZE = 10
+  net = CooCoo::Network.new()
+  net.layer(CooCoo::Layer.new(SIZE, SIZE / 2))
+  #net.layer(CooCoo::Layer.new(3, 3))
+  net.layer(CooCoo::Layer.new(SIZE / 2, SIZE / 2))
+  net.layer(CooCoo::Layer.new(SIZE / 2, 2))
+
+  inputs = 3.times.collect do |i|
+    CooCoo::Vector.zeros(SIZE)
+  end
+  inputs[0][0] = 1.0
+  inputs[1][2] = 1.0
+  inputs[2][3] = 1.0
+  targets = [ [ 1.0, 0.0 ],
+              [ 0.0, 1.0 ],
+              [ 0.0, 1.0 ]
+            ].collect do |v|
+    CooCoo::Vector[v]
+  end
+  
+  ENV.fetch('LOOPS', 100).to_i.times do |i|
+    targets.zip(inputs).each do |target, input|
+      net.learn(input, target, 0.3)
+    end
+  end
+
+  inputs.each.zip(targets) do |input, target|
+    output, hidden_state = net.forward(input)
+    err = (net.prep_input(target) - output.last)
+    puts("#{input} -> #{target}\t#{err}")
+    output.each_with_index do |o, i|
+      puts("\tLayer #{i}:\t#{o}")
+    end
+  end
+
+  puts(net.to_hash)
+end

data/lib/coo-coo/neuron.rb

@@ -0,0 +1,112 @@
+require 'coo-coo/consts'
+require 'coo-coo/debug'
+require 'coo-coo/math'
+require 'coo-coo/enum'
+require 'coo-coo/activation_functions'
+
+module CooCoo
+  class Neuron
+    def initialize(num_inputs, activation_func = CooCoo.default_activation)
+      @num_inputs = num_inputs
+      @activation_func = activation_func
+      @weights = @activation_func.initial_weights(num_inputs, 1)
+      @bias = @activation_func.initial_bias(1)[0]
+    end
+
+    def to_hash
+      { num_inputs: @num_inputs,
+        weights: @weights.to_a,
+        bias: @bias,
+        f: @activation_func.name
+      }
+    end
+
+    def update_from_hash!(h)
+      @num_inputs = h.fetch(:num_inputs, h.fetch(:weights, []).size)
+      @weights = CooCoo::Vector[h[:weights]]
+      @activation_func = CooCoo::ActivationFunctions.from_name(h[:f] || CooCoo.default_activation.name)
+      @bias = h.fetch(:bias, @activation_func.initial_bias(1)[0])
+      self
+    end
+
+    def self.from_hash(h)
+      self.new(h[:num_inputs] || h[:weights].size).update_from_hash!(h)
+    end
+    
+    attr_reader :num_inputs
+    attr_reader :weights
+    attr_reader :bias
+
+    def forward(input)
+      transfer(activate(input))
+    end
+
+    def activate(input)
+      (@weights * input).sum + @bias
+    end
+
+    def transfer(activation)
+      @activation_func.call(activation)
+    end
+    
+    def backprop(input, output, error)
+      # Properly: error * @activation_func.derivative(activate(input), output)
+      error * @activation_func.derivative(nil, output)
+    end
+
+    def transfer_error(delta)
+      @weights * delta
+    end
+
+    def weight_deltas(inputs, delta)
+      [ delta, inputs * delta ]
+    rescue
+      CooCoo.debug("#{$!}\n\t#{inputs.class}\t#{inputs}\n\t#{@weights.class}\t#{@weights}\n\t#{delta.class}\t#{delta}")
+      raise
+    end
+    
+    def update_weights!(inputs, delta)
+      adjust_weights!(*weight_deltas(inputs, delta))
+    end
+
+    def adjust_weights!(bias_delta, weight_deltas)
+      @bias -= bias_delta
+      @weights -= weight_deltas
+    end
+
+    def ==(other)
+      if other.kind_of?(self.class)
+        num_inputs == other.num_inputs && @weights == other.weights
+      else
+        false
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  require 'coo-coo/cost_functions'
+
+  n = CooCoo::Neuron.from_hash({ f: ENV.fetch("ACTIVATION", "Logistic"),
+                                 weights: [ 0.5, 0.5 ]
+                               })
+  inputs = [ CooCoo::Vector[[ 0.25, 0.75 ]], CooCoo::Vector[[ 0.0, 1.0 ]] ]
+  targets = [ 0.0, 1.0 ]
+  
+  ENV.fetch('LOOPS', 100).to_i.times do |i|
+    inputs.zip(targets).each do |input, target|
+      puts("#{i}: #{input} -> #{target}")
+      o = n.forward(input)
+      err1 = CooCoo::CostFunctions::MeanSquare.derivative(target, o)
+      puts("\tPre: #{input} * #{n.weights} = #{o}\t#{err1}\t#{CooCoo::CostFunctions::MeanSquare.call(target, o)}")
+      delta = n.backprop(input, o, err1)
+      puts("\tDelta: #{delta}")
+      n.update_weights!(input, delta * 0.3)
+      o = n.forward(input)
+      err2 = CooCoo::CostFunctions::MeanSquare.derivative(target, o)
+      puts("\tPost: #{input} * #{n.weights} = #{o}\t#{err2}")
+      puts("\tChange in Cost: #{err2} - #{err1} = #{err2 - err1}")
+      puts("")
+    end
+  end
+end