CooCoo 0.1.0

data/ext/buffer/matrix.pre.cu

@@ -0,0 +1,49 @@
+#include <stdio.h>
+#include <math.h>
+
+#include "public.h"
+#include "buffer.h"
+#include "matrix.h"
+
+#ifdef IN_PUBLIC
+typedef struct Matrix2D_s
+{
+  size_t width, height;
+  Buffer buffer;
+  double **rows;
+} *Matrix2D;
+#endif
+
+PUBLIC Matrix2D matrix_new(Buffer b, size_t w, size_t h)
+{
+  if(w * h >= b->length) {
+    return NULL;
+  }
+  
+  Matrix2D m = (Matrix2D)malloc(sizeof(Matrix2D_s));
+  m->width = w;
+  m->height = h;
+  m->buffer = b;
+  m->rows = (double **)malloc(sizeof(double) * h);
+  
+  for(size_t i = 0; i < h; i++) {
+    m->rows[i] = b->data + (i * w);
+  }
+
+  return m;
+}
+
+PUBLIC void matrix_free(Matrix2D m)
+{
+  for(size_t i = 0; i < m->height; i++) {
+    free(m->rows[i]);
+    m->rows[i] = NULL;
+  }
+
+  free(m);
+}
+
+PUBLIC Buffer matrix_buffer(Matrix2D m)
+{
+  return m->buffer;
+}

data/lib/CooCoo.rb

@@ -0,0 +1 @@
+require 'coo-coo'

data/lib/coo-coo.rb

@@ -0,0 +1,18 @@
+require 'coo-coo/platform'
+require 'coo-coo/consts'
+require 'coo-coo/math'
+require 'coo-coo/neuron'
+require 'coo-coo/layer'
+require 'coo-coo/network'
+require 'coo-coo/sequence'
+require 'coo-coo/temporal_network'
+require 'coo-coo/convolution'
+require 'coo-coo/recurrence/frontend'
+require 'coo-coo/trainer'
+require 'coo-coo/progress_bar'
+require 'coo-coo/activation_functions'
+require 'coo-coo/cost_functions'
+require 'coo-coo/option_parser'
+require 'coo-coo/generation_script'
+require 'coo-coo/drawing'
+require 'coo-coo/data_sources'

data/lib/coo-coo/activation_functions.rb

@@ -0,0 +1,344 @@
+require 'singleton'
+require 'coo-coo/from_name'
+
+module CooCoo
+  # Activation functions are functions of a single variable used by some
+  # {Layer}s to introduce non-linearities into or to alter data from a
+  # previous layer.
+  #
+  # To get an activation function instance use the included {#from_name}.
+  # From there you can call the methods found on the {Identity} activation
+  # function on any activation function.
+  #
+  # To create a new activation function that can be used in stored networks,
+  # you must subclass {Identity} and call {ActivationFunctions.register}.
+  module ActivationFunctions
+    class << self
+      include FromName
+    end
+
+    # The base for all the ActivationFunctions. Implements a do nothing
+    # activation function for a {Layer}.
+    class Identity
+      include Singleton
+      ActivationFunctions.register(self)
+
+      # Forwards missing class methods to the #instance.
+      def self.method_missing(mid, *args, &block)
+        instance.send(mid, *args, &block)
+      end
+
+      # A file friendly name for the activation function.
+      def name
+        self.class.name.split("::").last
+      end
+
+      def to_s
+        name
+      end
+
+      # Perform the activation.
+      # @param x [Numeric, Vector]
+      # @return [Numeric, Vector]
+      def call(x)
+        x
+      end
+
+      # Calculate the derivative at +x+.
+      # @param x [Numeric, Vector]
+      # @param y [Numeric, Vector, nil] Optional precomputed return value from #call.
+      def derivative(x, y = nil)
+        if (y || x).kind_of?(Numeric)
+          1.0
+        else
+          (y || x).class.ones((y || x).size)
+        end
+      end
+
+      # Initial weights a {Layer} should use when using this function.
+      # @param num_inputs [Integer] Number of inputs into the {Layer}
+      # @param size [Integer] The size or number of outputs of the {Layer}.
+      # @return [Vector] of weights that are randomly distributed
+      # between -1.0 and 1.0.
+      def initial_weights(num_inputs, size)
+        (CooCoo::Vector.rand(num_inputs * size) * 2.0 - 1.0) / num_inputs.to_f.sqrt
+      end
+
+      # Initial bias for a {Layer}.
+      # @param size [Integer] Number of bias elements to return.
+      # @return [Vector]
+      def initial_bias(size)
+        CooCoo::Vector.ones(size)
+      end
+
+      # Adjusts a {Network}'s inputs to the domain of the function.
+      # @param x [Vector]
+      # @return [Vector]
+      def prep_input(x)
+        x
+      end
+
+      # Adjusts a training set's target domain from +0..1+ to domain of the
+      # function's output.
+      # @param x [Vector]
+      # @return [Vector]
+      def prep_output_target(x)
+        x
+      end
+    end
+    
+    class Logistic < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        1.0 / ( 1.0 + (-x).exp)
+      end
+
+      def derivative(x, y = nil)
+        y ||= call(x)
+        y * (1.0 - y)
+      end
+    end
+
+    class TanH < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        2.0 / (1.0 + (x * -2.0).exp) - 1.0
+      end
+
+      def derivative(x, y = nil)
+        y ||= call(x)
+        1.0 - y * y
+      end
+
+      def initial_bias(size)
+        CooCoo::Vector.zeros(size)
+      end
+      
+      def prep_input(arr)
+        (arr.minmax_normalize(true) * 2.0) - 1.0
+      end
+
+      def prep_output_target(arr)
+        prep_input(arr)
+      end
+    end
+
+    class ReLU < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        t = x > 0
+        if t.kind_of?(FalseClass)
+          0.0
+        elsif t.kind_of?(TrueClass)
+          x
+        else
+          x * t
+        end
+      end
+
+      def derivative(x, y = nil)
+        y ||= call(x)
+        t = y > 0
+        if t.kind_of?(FalseClass)
+          0.0
+        elsif t.kind_of?(TrueClass)
+          1.0
+        else
+          t
+        end
+      end
+
+      def initial_weights(num_inputs, size)
+        CooCoo::Vector.rand(num_inputs * size) * (2.0 / (num_inputs * size).to_f).sqrt
+      end
+    end
+
+    class LeakyReLU < Identity
+      ActivationFunctions.register(self)
+      public_class_method :new
+      
+      def initialize(pos = 1.0, neg = 0.0001)
+        @positive_coeff = pos.to_f
+        @negative_coeff = neg.to_f
+      end
+
+      attr_accessor :positive_coeff
+      attr_accessor :negative_coeff
+      
+      def call(x)
+        pos = x > 0
+
+        if pos.kind_of?(FalseClass)
+          x * @negative_coeff
+        elsif pos.kind_of?(TrueClass)
+          x * @positive_coeff
+        else
+          neg = x <= 0
+          (x * pos * @positive_coeff) + (x * neg * @negative_coeff)
+        end
+      end
+
+      def derivative(x, y = nil)
+        y ||= call(x)
+        pos = y > 0
+        if pos.kind_of?(FalseClass)
+          @negative_coeff
+        elsif pos.kind_of?(TrueClass)
+          @positive_coeff
+        else
+          neg = y <= 0
+          (pos * @positive_coeff) + (neg * @negative_coeff)
+        end
+      end
+
+      def initial_weights(num_inputs, size)
+        CooCoo::Vector.rand(num_inputs * size) * (2.0 / (num_inputs * size).to_f).sqrt
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) &&
+          positive_coeff == other.positive_coeff &&
+          negative_coeff == other.negative_coeff
+      end
+    end
+
+    # Computes the Softmax function given a {Vector}:
+    #   y_i = e ** x_i / sum(e ** x)
+    # @see https://deepnotes.io/softmax-crossentropy
+    # @see https://becominghuman.ai/back-propagation-is-very-simple-who-made-it-complicated-97b794c97e5c
+    class SoftMax < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        e = x.exp
+        e / e.sum
+      end
+
+      def derivative(x, y = nil)
+        y ||= call(x)
+        s = x.exp.sum
+        y * (s - x) / s
+      end
+    end
+
+    # Computes the Softmax function given a {Vector} but subtracts the
+    # maximum value from every element prior to Softmax to prevent overflows:
+    #   y_i = e ** (x_i - max(x)) / sum(e ** (x - max(x)))
+    class ShiftedSoftMax < SoftMax
+      ActivationFunctions.register(self)
+
+      def call(x)
+        super(x - x.max)
+      end
+
+      def derivative(x, y = nil)
+        super(x - x.max, y)
+      end
+    end
+
+    class MinMax < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        if x.respond_to?(:minmax_normalize)
+          x.minmax_normalize
+        else
+          x
+        end
+      end
+
+      def derivative(x, y = nil)
+        min, max = x.minmax
+        (y || x).class.new((y || x).size, 1.0 / (max - min))
+      end
+
+      def prep_output_target(x)
+        x.minmax_normalize(true)
+      end
+    end
+
+    # Like the {MinMax} but safe when the input is all the same value.
+    class ZeroSafeMinMax < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        if x.respond_to?(:minmax_normalize)
+          x.minmax_normalize(true)
+        else
+          x
+        end
+      end
+
+      def derivative(x, y = nil)
+        min, max = x.minmax
+        delta = max - min
+        if delta == 0.0
+          x.zero
+        else
+          (y || x).class.new((y || x).size, 1.0 / (max - min))
+        end
+      end
+
+      def prep_output_target(x)
+        call(x)
+      end
+    end
+    
+    class Normalize < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        if x.respond_to?(:normalize)
+          x.normalize
+        else
+          x.coerce(0)
+        end
+      end
+
+      def derivative(x, y = nil)
+        mag = x.magnitude()
+        y ||= call(x)
+        1.0 / mag - y * y / mag
+      end
+
+      def prep_output_target(x)
+        x.normalize
+      end
+    end
+
+    # Like the {Normalize} but safe when the input is all the same value.
+    class ZeroSafeNormalize < Identity
+      ActivationFunctions.register(self)
+
+      def call(x)
+        if x.respond_to?(:normalize)
+          m = x.magnitude
+          if m == 0.0
+            0.0
+          else
+            x / magnitude
+          end
+        else
+          x.coerce(0)
+        end
+      end
+
+      def derivative(x, y = nil)
+        mag = x.magnitude()
+        if mag == 0.0
+          0.0
+        else
+          y ||= call(x)
+          1.0 / mag - y * y / mag
+        end
+      end
+
+      def prep_output_target(x)
+        x.normalize
+      end
+    end
+  end
+end

data/lib/coo-coo/consts.rb

@@ -0,0 +1,5 @@
+module CooCoo
+  def self.default_activation
+    CooCoo::ActivationFunctions::Logistic.instance
+  end
+end

data/lib/coo-coo/convolution.rb

@@ -0,0 +1,298 @@
+require 'coo-coo/layer_factory'
+
+module CooCoo
+  module Convolution
+    class BoxLayer
+      LayerFactory.register_type(self)
+
+      attr_reader :width
+      attr_reader :height
+      attr_reader :horizontal_step
+      attr_reader :vertical_step
+      attr_reader :input_width
+      attr_reader :input_height
+      attr_reader :int_output_width
+      attr_reader :int_output_height
+      attr_reader :internal_layer
+      attr_reader :delta_accumulator
+      
+      def initialize(width, height, horizontal_step, vertical_step, internal_layer, input_width, input_height, int_output_width, int_output_height, update_weights_with = :average)
+        @internal_layer = internal_layer
+        @width = width
+        @height = height
+        @horizontal_step = horizontal_step
+        @vertical_step = vertical_step
+        @input_width = input_width
+        @input_height = input_height
+        raise ArgumentError.new("Input size mismatch: #{input_width * input_height} is not #{internal_layer.num_inputs}") if internal_layer.num_inputs != (input_width * input_height)
+        @int_output_width = int_output_width
+        @int_output_height = int_output_height
+        raise ArgumentError.new("Input size mismatch: #{int_output_width * int_output_height} is not #{internal_layer.size}") if internal_layer.size != (int_output_width * int_output_height)
+        @delta_accumulator = delta_accumulator || :average
+        raise ArgumentError.new("Weights delta accumulator can only be averaged or summed") unless [ :average, :sum ].include?(@delta_accumulator)
+      end
+
+      def activation_function
+        internal_layer.activation_function
+      end
+
+      def horizontal_span
+        @horizontal_span ||= (@width / @horizontal_step.to_f).ceil
+      end
+
+      def vertical_span
+        @vertical_span ||= (@height / @vertical_step.to_f).ceil
+      end
+      
+      def num_inputs
+        @width * @height
+      end
+
+      def output_width
+        (horizontal_span * int_output_width).to_i
+      end
+
+      def output_height
+        (vertical_span * int_output_height).to_i
+      end
+
+      def size
+        output_height * output_width
+      end
+
+      def neurons
+        internal_layer.neurons
+      end
+
+      def flatten_areas(outputs, w, h, inner_width)
+        out = CooCoo::Vector.new(w * h)
+        
+        each_area do |grid_x, grid_y|
+          area_output = outputs[grid_y][grid_x]
+          gx = grid_x * w / horizontal_span.to_f
+          gy = grid_y * h / vertical_span.to_f
+          out.set2d!(w, area_output, inner_width, gx, gy)
+        end
+
+        out
+      end
+      
+      def forward(input, hidden_state)
+        hs = hidden_state[self] || Array.new
+        outputs = each_area do |grid_x, grid_y|
+          hs_index = (grid_y * horizontal_span + grid_x).to_i
+          output, layer_hs = @internal_layer.forward(slice_input(input, grid_x, grid_y), hs[hs_index])
+          hs[hs_index] = layer_hs
+          output
+        end
+        hidden_state[self] = hs
+        [ flatten_areas(outputs, horizontal_span * int_output_width, vertical_span * int_output_height, int_output_width), hidden_state ]
+      end
+
+      def backprop(input, output, errors, hidden_state)
+        hs = hidden_state[self] || Array.new
+        deltas = each_area do |grid_x, grid_y|
+          hs_index = grid_y * horizontal_span + grid_x
+          d, layer_hs = @internal_layer.backprop(slice_input(input, grid_x, grid_y), slice_output(output, grid_x, grid_y), slice_output(errors, grid_x, grid_y), hs[hs_index])
+          hs[hs_index] = layer_hs
+          d
+        end
+        hidden_state[self] = hs
+        [ Sequence[deltas.collect { |d| Sequence[d] }], hidden_state ]
+      end
+
+      def transfer_error(deltas)
+        flatten_areas(each_area do |grid_x, grid_y|
+                        @internal_layer.transfer_error(deltas[grid_y][grid_x]).to_a
+                      end, width, height, input_width)
+      end
+
+      def update_weights!(inputs, deltas)
+        adjust_weights!(*weight_deltas(inputs, deltas))
+      end
+
+      def adjust_weights!(deltas)
+        @internal_layer.adjust_weights!(deltas)
+        self
+      end
+
+      def weight_deltas(inputs, deltas)
+        #rate = rate / (@horizontal_span * @vertical_span).to_f
+        change = []
+        wd = []
+
+        d = []
+        each_area do |grid_x, grid_y|
+          hs_index = grid_y * horizontal_span + grid_x
+          delta, hs = @internal_layer.
+            weight_deltas(slice_input(inputs, grid_x, grid_y),
+                          deltas[grid_y][grid_x])
+          d << delta
+        end
+
+        Sequence[d].send(@delta_accumulator)
+      end
+
+      def ==(other)
+        other.kind_of?(self.class) &&
+          width == other.width &&
+          height == other.height &&
+          horizontal_step == other.horizontal_step &&
+          vertical_step == other.vertical_step &&
+          input_width == other.input_width &&
+          input_height == other.input_height &&
+          int_output_width == other.int_output_width &&
+          int_output_height == other.int_output_height &&
+          internal_layer == other.internal_layer &&
+          delta_accumulator == other.delta_accumulator
+      end
+
+      def to_hash(network = nil)
+        { type: self.class.to_s,
+          width: @width,
+          height: @height,
+          horizontal_step: @horizontal_step,
+          vertical_step: @vertical_step,
+          input_width: @input_width,
+          input_height: @input_height,
+          int_output_width: @int_output_width,
+          int_output_height: @int_output_height,
+          delta_accumulator: @delta_accumulator,
+          internal_layer: @internal_layer.to_hash(network)
+        }
+      end
+
+      def self.from_hash(h, network = nil)
+        self.new(h.fetch(:width), h.fetch(:height),
+                 h.fetch(:horizontal_step), h.fetch(:vertical_step),
+                 LayerFactory.from_hash(h.fetch(:internal_layer)),
+                 h.fetch(:input_width), h.fetch(:input_height),
+                 h.fetch(:int_output_width), h.fetch(:int_output_height),
+                 h.fetch(:delta_accumulator, :average))
+      end
+
+      #private
+
+      def each_area
+        return to_enum(:each_area) unless block_given?
+
+        vertical_span.to_i.times.collect do |grid_y|
+          horizontal_span.to_i.times.collect do |grid_x|
+            yield(grid_x, grid_y)
+          end
+        end
+      end
+      
+      def slice_input(input, grid_x, grid_y)
+        origin_x = grid_x * @horizontal_step
+        origin_y = grid_y * @vertical_step
+        input.slice_2d(@width,
+                       @height,
+                       origin_x, origin_y,
+                       @input_width, @input_height,
+                       0.0)
+      end
+      
+      def slice_output(output, grid_x, grid_y)
+        origin_x = grid_x * @int_output_width
+        origin_y = grid_y * @int_output_height
+        output.slice_2d((horizontal_span * @int_output_width).to_i,
+                        (vertical_span * @int_output_height).to_i,
+                        origin_x, origin_y,
+                        @int_output_width, @int_output_height,
+                        0.0)
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  require 'coo-coo/layer'
+  require 'coo-coo/cost_functions'
+  
+  WIDTH = 16
+  HEIGHT = 16
+  X_STEP = 4
+  Y_STEP = 4
+  CONV_WIDTH = 4
+  CONV_HEIGHT = 4
+  CONV_OUT_WIDTH = 1
+  CONV_OUT_HEIGHT = 1
+  activation = CooCoo::ActivationFunctions.from_name(ENV.fetch('ACTIVATION', 'Logistic'))
+  cost_function = CooCoo::CostFunctions.from_name(ENV.fetch('COST', 'MeanSquare'))
+  
+  inner_layer = CooCoo::Layer.new(CONV_WIDTH * CONV_HEIGHT, CONV_OUT_WIDTH * CONV_OUT_HEIGHT, activation)
+  layer = CooCoo::Convolution::BoxLayer.new(WIDTH, HEIGHT, X_STEP, Y_STEP, inner_layer, CONV_WIDTH, CONV_HEIGHT, CONV_OUT_WIDTH, CONV_OUT_HEIGHT)
+
+  INPUT_SIZE = layer.num_inputs
+  OUT_WIDTH = layer.output_width
+  OUT_HEIGHT = layer.output_height
+  OUTPUT_SIZE = layer.size
+  learning_rate = ENV.fetch('RATE', 0.3).to_f
+  
+  input = [ 1.0 ] + (INPUT_SIZE - 2).times.collect { 0.0 } + [ 1.0 ]
+  input = CooCoo::Vector[input, INPUT_SIZE]
+  target = CooCoo::Vector.zeros(OUTPUT_SIZE)
+  target[0] = 1.0
+  target[-1] = 1.0
+
+  input = activation.prep_input(input)
+  target = activation.prep_input(target)
+
+  #input = (input - 0.5) * 2.0
+  #target = (target - 0.5) * 2.0
+
+  def matrix_image(m, width)
+    puts("matrix image #{width}")
+    s = m.to_a.each_slice(width).collect do |line|
+      line.collect do |c|
+        if c > 0.75
+          '#'
+        elsif c > 0.5
+          'X'
+        elsif c > 0.25
+          'x'
+        elsif c >= 0.0
+          '.'
+        elsif c >= -0.5
+          '-'
+        else
+          '~'
+        end
+      end.join
+    end.join("\n")
+  end
+
+  require 'benchmark'
+  
+  Benchmark.bm(3) do |bm|
+    bm.report("loops") do
+      ENV.fetch("LOOPS", 100).to_i.times do |i|
+        puts("#{i}\n========\n")
+        #puts("Inputs =\n#{matrix_image(input, WIDTH)}")
+        output, hs = layer.forward(input, {})
+        #puts("Output = #{output}\n#{matrix_image(output, OUT_WIDTH)}")
+        err = cost_function.derivative(target, output)
+        #puts("Target = #{target}\n#{matrix_image(target, OUT_WIDTH)}")
+        #puts("Err = #{err}\n#{matrix_image(err * 10.0, OUT_WIDTH)}")
+        puts("|Err| = #{err.magnitude} #{(err * err).magnitude}")
+        deltas, hs = layer.backprop(input, output, err, hs)
+        #puts("Deltas = #{deltas}\n#{matrix_image(deltas, OUT_WIDTH)}")
+        xfer = layer.transfer_error(deltas)
+        #puts("Xfer error = #{xfer}\n#{matrix_image(xfer, OUT_WIDTH)}")
+        layer.update_weights!(input, deltas * learning_rate)
+        #puts("Weights updated")
+        output, hs = layer.forward(input, {})
+        puts("New output = #{output}\n#{matrix_image(output, OUT_WIDTH)}")
+      end
+    end
+  end
+  
+  # layer.each_area do |x, y|
+  #   puts("#{x}, #{y}\t#{x * CONV_WIDTH}, #{y * CONV_HEIGHT}")
+  #   puts(matrix_image(layer.slice_input(input, x, y), CONV_WIDTH))
+  #   puts
+  #   puts(matrix_image(layer.slice_output(target, x, y), CONV_OUT_WIDTH))
+  #   puts
+  # end
+end