CooCoo 0.1.0

data/lib/coo-coo/trainer/batch_stats.rb

@@ -0,0 +1,27 @@
+module CooCoo
+  module Trainer
+    class BatchStats
+      attr_reader :trainer
+      attr_reader :batch
+      attr_reader :batch_size
+      attr_reader :total_time
+      attr_reader :total_loss
+      
+      def initialize(trainer, batch, batch_size, total_time, total_loss)
+        @trainer = trainer
+        @batch = batch
+        @batch_size = batch_size
+        @total_time = total_time
+        @total_loss = total_loss
+      end
+
+      def average_time
+        total_time / batch_size.to_f
+      end
+
+      def average_loss
+        total_loss / batch_size.to_f
+      end
+    end
+  end
+end

data/lib/coo-coo/trainer/momentum_stochastic.rb

@@ -0,0 +1,59 @@
+require 'coo-coo/cost_functions'
+require 'coo-coo/sequence'
+require 'coo-coo/trainer/base'
+require 'coo-coo/trainer/batch_stats'
+
+module CooCoo
+  module Trainer
+    class MomentumStochastic < Base
+      DEFAULT_OPTIONS = Base::DEFAULT_OPTIONS.merge(momentum: 1/30.0)
+      
+      def options
+        super(DEFAULT_OPTIONS) do |o, options|
+          o.on('--momentum FLOAT', Float, 'Multiplier for the accumulated changes.') do |n|
+            options.momentum = n
+          end
+        end
+      end
+      
+      # @option options [Float] :momentum The dampening factor on the reuse of the previous network change.
+      def train(options, &block)
+        options = options.to_h
+        network = options.fetch(:network)
+        training_data = options.fetch(:data)
+        learning_rate = options.fetch(:learning_rate, 1/3.0)
+        batch_size = options.fetch(:batch_size, 1024)
+        cost_function = options.fetch(:cost_function, CostFunctions::MeanSquare)
+        momentum = options.fetch(:momentum, 1/30.0)
+
+        t = Time.now
+        
+        training_data.each_slice(batch_size).with_index do |batch, i|
+          last_delta = 0.0
+          total_errs = batch.inject(nil) do |acc, (expecting, input)|
+            errs, hidden_state, last_delta = learn(network, input, expecting, learning_rate, last_delta, momentum, cost_function, Hash.new)
+            errs + (acc || 0)
+          end
+
+          if block
+            block.call(BatchStats.new(self, i, batch_size, Time.now - t, total_errs))
+          end
+          
+          t = Time.now
+        end
+      end
+
+      def learn(network, input, expecting, rate, last_deltas, momentum, cost_function, hidden_state)
+        output, hidden_state = network.forward(input, hidden_state)
+        target = expecting
+        target = network.prep_output_target(expecting)
+        final_output = network.final_output(output)
+        errors = cost_function.derivative(target, final_output)
+        deltas, hidden_state = network.backprop(input, output, errors, hidden_state)
+        deltas = CooCoo::Sequence[deltas] * rate
+        network.update_weights!(input, output, deltas - last_deltas * momentum)
+        return cost_function.call(target, final_output), hidden_state, deltas
+      end
+    end
+  end
+end

data/lib/coo-coo/trainer/stochastic.rb

@@ -0,0 +1,47 @@
+require 'coo-coo/cost_functions'
+require 'coo-coo/sequence'
+require 'coo-coo/trainer/base'
+require 'coo-coo/trainer/batch_stats'
+
+module CooCoo
+  module Trainer
+    # Implements straight up stochastic gradient descent. No alterations
+    # get made to any hyperparameters while learning happens after every
+    # example.
+    class Stochastic < Base
+      def train(options, &block)
+        options = options.to_h
+        network = options.fetch(:network)
+        training_data = options.fetch(:data)
+        learning_rate = options.fetch(:learning_rate, 0.3)
+        batch_size = options.fetch(:batch_size, 1024)
+        cost_function = options.fetch(:cost_function, CostFunctions::MeanSquare)
+        
+        t = Time.now
+        
+        training_data.each_slice(batch_size).with_index do |batch, i|
+          total_errs = batch.inject(nil) do |acc, (expecting, input)|
+            errs, hidden_state = learn(network, input, expecting, learning_rate, cost_function, Hash.new)
+            errs + (acc || 0)
+          end
+
+          if block
+            block.call(BatchStats.new(self, i, batch.size, Time.now - t, total_errs))
+          end
+          
+          t = Time.now
+        end
+      end
+
+      def learn(network, input, expecting, rate, cost_function = CostFunctions::MeanSquare, hidden_state)
+        output, hidden_state = network.forward(input, hidden_state)
+        target = network.prep_output_target(expecting)
+        final_output = network.final_output(output)
+        errors = cost_function.derivative(target, final_output)
+        deltas, hidden_state = network.backprop(input, output, errors, hidden_state)
+        network.update_weights!(input, output, deltas * rate)
+        return cost_function.call(target, final_output), hidden_state
+      end
+    end
+  end
+end

data/lib/coo-coo/transformer.rb

@@ -0,0 +1,272 @@
+require 'nmatrix'
+
+module CooCoo
+  module Transformers
+    class Base < Enumerator
+      def drop(n)
+        n.times do
+          self.next
+        end
+        
+        self
+      rescue StopIteration
+        self
+      end
+
+      def first(n)
+        Stopper.new(self, n)
+      end
+
+      def self.bin_op(*ops)
+        ops.each do |op|
+          bin_op_inner(op)
+        end
+      end
+
+      def self.bin_op_inner(op)
+        define_method(op) do |other|
+          Combo.new(self, other) do |a, b|
+            a.send(op, b)
+          end
+        end
+      end
+
+      bin_op :+, :-, :*, :/
+    end
+
+    class Proxy < Base
+      def initialize(enum)
+        @enum = enum
+        
+        super() do |yielder|
+          loop do
+            yielder << self.next
+          end
+        end
+      end
+
+      def next
+        @enum.next
+      end
+    end
+    
+    class Stopper < Proxy
+      def initialize(enum, n)
+        @stop_after = n
+        @index = 0
+
+        super(enum)
+      end
+
+      def next
+        if @index < @stop_after
+          @index += 1
+          super
+        else
+          raise StopIteration
+        end
+      end
+    end
+
+    class Combo < Base
+      def initialize(src, other, &op)
+        @src = src
+        @other = other
+        @op = op
+        
+        super() do |yielder|
+          loop do
+            yielder << self.next
+          end
+        end
+      end
+
+      def next
+        @op.call(@src.next, @other.next)
+      end
+    end
+
+    module Image
+      class Base < ::CooCoo::Transformers::Proxy
+        def initialize(enum, width, height)
+          @width = width
+          @height = height
+          super(enum)
+        end
+
+        attr_reader :width, :height
+        
+        def translate(x, y)
+          Translation.new(self, width, height, x, y)
+        end
+
+        def rotate(radians, ox = 0, oy = 0)
+          Rotation.new(self, width, height, ox, oy, radians)
+        end
+
+        def scale(x, y)
+          Scaler.new(self, width, height, x, y || x)
+        end
+      end
+
+      class Translation < Base
+        def initialize(enum, width, height, tx, ty)
+          super(enum, width, height)
+          @tx = tx
+          @ty = ty
+        end
+
+        def next
+          i = super()
+          r = NMatrix.zeroes([1, width * height])
+          height.times do |y|
+            width.times do |x|
+              r[0, map_pixel(x, y)] = i[0, map_pixel(*translate_pixel(x, y))]
+            end
+          end
+          r
+        end
+
+        private
+        
+        def map_pixel(x, y)
+          x + y * width
+        end
+        
+        def translate_pixel(x, y)
+          [ (x + @tx) % width, (y + @ty) % height ]
+        end
+      end
+      
+      class Rotation < Base
+        def initialize(enum, width, height, ox, oy, radians)
+          super(enum, width, height)
+          @ox = ox
+          @oy = oy
+          @radians = radians
+        end
+
+        def next
+          i = super()
+          r = NMatrix.zeroes([1, width * height])
+          height.times do |y|
+            width.times do |x|
+              r[0, map_pixel(x, y)] = sample(i, x, y)
+            end
+          end
+          r
+        end
+
+        private
+
+        def sample(image, x, y)
+          rx, ry = *rotate_pixel(x, y)
+          rx_min = rx.floor % width
+          rx_max = rx.ceil % width
+          ry_min = ry.floor % height
+          ry_max = ry.ceil % height
+          
+          (image[0, map_pixel(rx_min, ry_min)] +
+           image[0, map_pixel(rx_max, ry_min)] +
+           image[0, map_pixel(rx_min, ry_max)] +
+           image[0, map_pixel(rx_max, ry_max)]) / 4.0
+        end
+        
+        def map_pixel(x, y)
+          x + y * width
+        end
+
+        def rotate_pixel(x, y)
+          c = Math.cos(@radians)
+          s = Math.sin(@radians)
+          x = (x - @ox)
+          x = x * c - y * s
+          y = y - @oy
+          y = x * s + y * c
+          [ x, y ]
+        end
+      end
+
+      class Scaler < Base
+        def initialize(enum, width, height, scale_x, scale_y)
+          super(enum, width, height)
+          @scale_x = scale_x
+          @scale_y = scale_y
+        end
+
+        def next
+          i = super()
+          r = NMatrix.zeroes([1, width * height])
+          height.times do |y|
+            width.times do |x|
+              r[0, map_pixel(x, y)] = sample(i, x, y)
+            end
+          end
+          r
+        end
+
+        private
+
+        def sample(image, x, y)
+          if @scale_x == 1.0 && @scale_y == 1.0
+            image[0, map_pixel(x, y)]
+          elsif @scale_x > 1.0 && @scale_y > 1.0
+            rx, ry = *scale_pixel(x, y)
+            image[0, map_pixel(rx.to_i, ry.to_i)]
+          else
+            rx, ry = *scale_pixel(x, y)
+            rx_min = rx.floor % width
+            rx_max = rx.ceil % width
+            ry_min = ry.floor % height
+            ry_max = ry.ceil % height
+            
+            (image[0, map_pixel(rx_min, ry_min)] +
+             image[0, map_pixel(rx_max, ry_min)] +
+             image[0, map_pixel(rx_min, ry_max)] +
+             image[0, map_pixel(rx_max, ry_max)]) / 4.0
+          end
+        end
+        
+        def map_pixel(x, y)
+          x + y * width
+        end
+
+        def scale_pixel(x, y)
+          [ (x * @scale_x) % width, (y * @scale_y) % height ]
+        end
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  ra = (0...Float::INFINITY)
+  data = CooCoo::Transformers::Proxy.new(ra.each)
+  data2 = CooCoo::Transformers::Proxy.new(ra.each)
+  data3 = CooCoo::Transformers::Proxy.new(ra.each)
+  puts("Raw data: #{ra.first(10).inspect}")
+  t = data * data2
+  r = t.first(10)
+  puts("Squared: #{r.to_a.inspect} #{r.class}")
+  r = (t + 1000) / data3.drop(1) - 2
+  r = r.first(10)
+  puts("Add 1000: #{r.to_a.inspect} #{r.class}")
+  #puts(t.each.to_a.inspect)
+
+  i = Array.new(8, [1.0] + [0.0] * 7)
+  i[0] = [1.0] * 8
+  i[7] = i[0]
+  puts(*i.each_slice(8).to_a[0].collect(&:inspect))
+  
+  e = Enumerator.new do |y|
+    y << i.flatten.to_nm([1, 64])
+  end
+
+  t = CooCoo::Transformers::Image::Base.new(e, 8, 8).
+        #rotate(Math::PI / 4.0).
+        scale(0.5, 0.5).
+        translate(2, 1)
+  i = t.next
+  puts()
+  puts(*i.each_slice(8).to_a.collect(&:inspect))
+end

data/lib/coo-coo/vector_layer.rb

@@ -0,0 +1,194 @@
+require 'coo-coo/consts'
+require 'coo-coo/math'
+require 'coo-coo/debug'
+require 'coo-coo/layer_factory'
+require 'coo-coo/activation_functions'
+require 'coo-coo/weight_deltas'
+
+module CooCoo
+  class VectorLayer
+    LayerFactory.register_type(self)
+    
+    attr_accessor :activation_function
+    attr_reader :bias
+    attr_reader :weights
+    
+    def initialize(num_inputs, size, activation_function = CooCoo.default_activation)
+      @activation_function = activation_function
+      @num_inputs = num_inputs.to_i
+      @size = size.to_i
+      @weights = @activation_function.initial_weights(@num_inputs, @size)
+      @bias = @activation_function.initial_bias(@size)
+    end
+
+    def num_inputs
+      @num_inputs
+    end
+
+    def size
+      @size
+    end
+
+    def forward(input, hidden_state)
+      return transfer(activate(input)), hidden_state
+    end
+
+    def transfer(activations)
+      @activation_function.call(activations)
+    end
+
+    def activate(input)
+      @weights.dot(num_inputs, size, input, 1, num_inputs) + @bias
+    end
+
+    def backprop(input, output, errors, hidden_state)
+      # Properly: return errors * @activation_func.derivative(activate(input), output), hidden_state
+      return errors * @activation_function.derivative(nil, output), hidden_state
+    end
+
+    def transfer_error(deltas)
+      deltas.dot(size, 1, @weights, num_inputs, size)
+    end
+
+    def transfer_input_error(expecting)
+      (output - expecting).to_a
+    end
+
+    def update_weights!(inputs, deltas)
+      adjust_weights!(weight_deltas(inputs, deltas))
+    end
+
+    def adjust_weights!(deltas)
+      @bias -= deltas.bias_deltas
+      @weights -= deltas.weight_deltas
+
+      self
+    end
+
+    def weight_deltas(inputs, deltas)
+      WeightDeltas.new(deltas, deltas.dot(1, size, inputs, num_inputs, 1))
+    end
+
+    def to_hash(network = nil)
+      { type: self.class.to_s,
+        outputs: size,
+        neurons: neuron_hash
+      }
+    end
+
+    def neuron_hash
+      @weights.each_slice(num_inputs).with_index.collect do |neuron_weights, i|
+        { num_inputs: num_inputs,
+          weights: neuron_weights.to_a,
+          bias: @bias[i],
+          f: @activation_function.name
+        }      
+      end
+    end
+
+    def add_neurons!(new_size)
+      if new_size != @size
+        w = CooCoo::Vector.zeros(num_inputs * new_size)
+        w[0, @weights.size] = @weights
+        w[@weights.size, num_inputs] = @activation_func.initial_weights(num_inputs, 1)
+        @weights = w
+
+        @bias = CooCoo::Vector.ones(new_size).set(@bias)
+        @bias[-1] = @activation_func.initial_bias(1)[0]
+
+        @size = new_size
+      end
+      
+      self
+    end
+
+    def add_inputs!(new_size)
+      if new_size != num_inputs
+        w = CooCoo::Vector.zeros(new_size * size)
+        w.set2d!(new_size, @weights, num_inputs, 0, 0)
+        w.set2d!(new_size, @activation_func.initial_weights(size, 1), 1, new_size - 1, 0)
+        @weights = w
+        @num_inputs = new_size
+      end
+      
+      self
+    end
+
+    def update_neuron_from_hash!(neuron_index, h)
+      if neuron_index > size
+        add_neurons!(neuron_index)
+      end
+
+      @weights[neuron_index * num_inputs, num_inputs] = h[:weights]
+      @bias[neuron_index] = h[:bias]
+    end
+
+    def update_from_hash!(h)
+      @activation_function = ActivationFunctions.from_name(h[:neurons][0][:f])
+      add_neurons!(h[:outputs])
+      add_inputs!(h[:neurons][0][:num_inputs])
+      
+      h[:outputs].times do |i|
+        update_neuron_from_hash!(i, h[:neurons][i])
+      end
+
+      self
+    end
+
+    def ==(other)
+      other.kind_of?(self.class) &&
+        size == other.size &&
+        bias == other.bias &&
+        weights == other.weights &&
+        activation_function == other.activation_function
+    end
+    
+    class << self
+      def from_hash(h, network = nil)
+        self.new(h[:neurons][0][:num_inputs],
+                 h[:outputs],
+                 ActivationFunctions.from_name(h[:neurons][0][:f])).
+          update_from_hash!(h)
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  layer = CooCoo::VectorLayer.new(4, 2, CooCoo::ActivationFunctions.from_name(ENV.fetch("ACTIVATION", "Logistic")))
+  inputs = [ [ 1.0, 0.0, 0.0, 0.0 ],
+             [ 0.0, 0.0, 1.0, 0.0 ],
+             [ 0.0, 1.0, 0.0, 0.0],
+             [ 0.0, 0.0, 0.0, 1.0 ]
+           ].collect do |v|
+    CooCoo::CUDA::Vector[v]
+  end
+  targets = [ [ 1.0, 0.0 ],
+              [ 0.0, 1.0 ],
+              [ 0.0, 0.0 ],
+              [ 0.0, 0.0 ]
+            ].collect do |v|
+    CooCoo::CUDA::Vector[v]
+  end
+
+  inputs.zip(targets).cycle(ENV.fetch('LOOPS', 100).to_i).each do |(input, target)|
+    output, hidden_state = layer.forward(input, Hash.new)
+    puts("#{input} -> #{target} #{target.inspect}")
+    puts("\toutput: #{output}")
+
+    err = (output - target)
+    puts("\terr: #{err}")
+    #err = err * err * 0.5
+    delta, hidden_state = layer.backprop(input, output, err, hidden_state)
+    puts("\tdelta: #{delta}")
+    puts("\terror: #{err}")
+    puts("\txfer: #{layer.transfer_error(delta)}")
+
+    layer.update_weights!(input, delta * 0.5)
+  end
+
+  inputs.zip(targets).each do |(input, target)|
+    output, hidden_state = layer.forward(input, Hash.new)
+    puts("#{input} -> #{output}\t#{target}")
+  end
+end