CooCoo 0.1.0

data/lib/coo-coo/core_ext.rb

@@ -0,0 +1,75 @@
+class Numeric
+  [ :exp, :sqrt, :log, :log10, :log2,
+    :sin, :asin, :cos, :acos, :tan, :atan,
+    :sinh, :asinh, :cosh, :acosh, :tanh, :atanh,
+    :ceil, :floor, :round
+  ].each do |f|
+    define_method(f) do
+      ::Math.send(f, self)
+    end
+  end
+
+  def identity
+    coerce(1)[0]
+  end
+
+  def zero
+    coerce(0)[0]
+  end
+end
+
+class Object
+  def self.instance_defines?(method)
+    instance_methods.include?(method)
+  end
+
+  def self.define_once(method, &definition)
+    unless instance_defines?(method)
+      define_method(method, &definition)
+    end
+  end
+
+  def self.delegate(*args)
+    opts = args.pop
+    
+    args.each do |meth|
+      define_method(meth) do |*a|
+        send(opts[:to]).send(meth, *a)
+      end
+    end
+  end
+end
+
+class Array
+  def zero
+    self.class.new(size, 0.0)
+  end
+end
+
+class File
+  def self.write_to(path, &block)
+    tmp = path.to_s + ".tmp"
+    bak = path.to_s + "~"
+
+    # write to temp file
+    File.open(tmp, "w", &block)
+
+    # create a backup file
+    if File.exists?(path)
+      # remove any existing backup
+      if File.exists?(bak)
+        File.delete(bak)
+      end
+
+      File.rename(path, bak)
+    end
+
+    # finalize the save
+    File.rename(tmp, path)
+    
+    self
+  rescue
+    File.delete(tmp)
+    raise
+  end
+end

data/lib/coo-coo/cost_functions.rb

@@ -0,0 +1,91 @@
+require 'coo-coo/from_name'
+
+module CooCoo
+  # CostFunctions are used with a {Trainer} to determine how close a {Network}
+  # is coming to its target. CostFunctions are functions of two variables.
+  #
+  # To get a cost function instance use the included {#from_name}.
+  # Then you can +#call+ or +#derivative+ any cost function.
+  #
+  # To create a new cost function that can be used with a {Trainer},
+  # you must call {CostFunctions.register} and implement the
+  # +#call+ and +#derivative+ class methods.
+  module CostFunctions
+    class << self
+      include FromName
+    end
+
+    # @abstract Defines and documents the cost functions' interface.
+    # Be sure to call {CostFunctions.register} inside your subclass.
+    class Base
+      # Returns the cost between the target output and actual output.
+      #
+      # @param target [Vector] Desired value
+      # @param x [Vector] A network's actual output
+      # @return [Vector] The cost of the target for this output
+      def self.call(target, x)
+        raise NotImplementedError.new
+      end
+
+      # Returns the derivative of the cost function, +#call+. This is
+      # what gets fed into the network to determine the changes.
+      #
+      # @param target [Vector] Desired value
+      # @param x [Vector] A network's actual output
+      # @param y [Vector] The results from a previous +#call+
+      # @return [Vector]
+      def self.derivative(target, x, y = nil)
+        raise NotImplementedError.new
+      end
+    end
+
+    # Implements the mean square cost function. Its derivative is
+    # a simple difference between the target and actual output.
+    class MeanSquare < Base
+      CostFunctions.register(self, name)
+      
+      def self.call(target, x)
+        d = derivative(target, x)
+        d * d * 0.5
+      end
+
+      def self.derivative(target, x, y = nil)
+        x - target
+      end
+    end
+
+    # Implements the log cross-entropy cost function that is used with
+    # {ActivationFunctions::SoftMax} and
+    # {ActivationFunctions::ShiftedSoftMax}. This calls +Math.log+ on
+    # the network's output and multiples that by the target. Therefore
+    # good target values are +0...1+.
+    class CrossEntropy < Base
+      CostFunctions.register(self, name)
+      
+      def self.call(target, x)
+        -x.log * target
+      end
+
+      def self.derivative(target, x)
+        -target / x
+      end
+    end
+
+    # Combines a SoftMax activation with CrossEntropy. Due to math this
+    # is more optimal than having a SoftMax layer and doing CrossEntropy
+    # seperately.
+    #
+    # @see http://peterroelants.github.io/posts/neural_network_implementation_intermezzo02/
+    class SoftMaxCrossEntropy < CrossEntropy
+      CostFunctions.register(self, name)
+      
+      def self.call(target, x)
+        super(target, ActivationFunctions::ShiftedSoftMax.call(x))
+      end
+
+      def self.derivative(target, x)
+        x - target
+      end
+    end
+  end
+end

data/lib/coo-coo/cuda.rb

@@ -0,0 +1,116 @@
+require 'coo-coo/debug'
+begin
+  require 'coo-coo/cuda/runtime'
+
+  module CooCoo
+    module CUDA
+      def self.available?
+        ENV["COOCOO_USE_CUDA"] != "0"# && Runtime.device_count > 0
+      end
+
+      def self.memory_info
+        Runtime.memory_info
+      end
+      
+      def self.collect_garbage(size = nil)
+        free, total = memory_info
+        if size == nil || (3 * size + free) >= total
+          GC.start
+          new_free, total = memory_info
+          diff = free - new_free
+          if size && (size + new_free) >= total
+            raise NoMemoryError.new(size)
+          end
+        end
+      end
+    end
+  end
+
+  require 'coo-coo/cuda/host_buffer'
+  require 'coo-coo/cuda/device_buffer'
+  require 'coo-coo/cuda/vector'
+
+rescue LoadError
+  CooCoo.debug("LoadError #{__FILE__}: #{$!}")
+  module CooCoo
+    module CUDA
+      def self.available?
+        false
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  require 'pp'
+
+  puts("Cuda not available") unless CooCoo::CUDA.available?
+  
+  puts("Resetting #{CooCoo::CUDA::Runtime.cudaDeviceReset}")
+  puts("Device = #{CooCoo::CUDA::Runtime.get_device} / #{CooCoo::CUDA::Runtime.device_count}")
+  puts("Init = #{CooCoo::CUDA::DeviceBuffer::FFI.buffer_init(0)}")
+  puts("Block size = #{CooCoo::CUDA::DeviceBuffer::FFI.buffer_block_size}")
+  puts("Grid size = #{CooCoo::CUDA::DeviceBuffer::FFI.buffer_max_grid_size}")
+  puts("Total memory = #{CooCoo::CUDA.memory_info.join('/')}")
+  props = CooCoo::CUDA::Runtime::DeviceProperties.new
+  puts(CooCoo::CUDA::Runtime.cudaGetDeviceProperties(props, 0).inspect)
+  puts("Properties")
+  props.members.each do |m|
+    value = props[m]
+    if m != :name && value.kind_of?(FFI::Struct::InlineArray)
+      value.each_with_index do |v, i|
+        puts("#{m}[#{i}]\t#{v}")
+      end
+    else
+      puts("#{m}\t#{value}")
+    end
+  end
+  dev = CooCoo::CUDA::Runtime.get_device
+  puts("Device #{dev}")
+  puts("Creating")
+  WIDTH = 256
+  HEIGHT = 256
+  SIZE = WIDTH * HEIGHT # 1024 * 1024 * 1
+  h = CooCoo::CUDA::HostBuffer.new(SIZE)
+  arr = SIZE.times.collect { |n| n }
+  h.set(arr)
+  a = CooCoo::CUDA::Vector.new(SIZE)
+  a.set(h)
+  puts("Size = #{a.size}")
+  puts("Getting")
+  b = ((a.dot(WIDTH, HEIGHT, a) * 3 - a) / 3.0).sin #* 2 + 1
+  #b = b.get.to_a
+  puts(b[0, 10].to_s)
+  puts(b[-10, 10].to_s)
+  puts("Sum = #{b.sum} #{b.each.sum}")
+
+  require 'benchmark'
+  require 'coo-coo/math'
+  require 'nmatrix'
+  
+  Benchmark.bm(3) do |bm|
+    bm.report("cuda add") do
+      b = a.clone
+      10000.times do |i|
+        #puts("%i %i" % [ CooCoo::CUDA::DeviceBuffer::FFI.buffer_total_bytes_allocated, CooCoo::CUDA::Runtime.total_global_mem ]) if i % 1000
+        b = b + b
+      end
+      #puts("CUDA sum", b.get.to_a.inspect)
+      #puts("Last error: ", CooCoo::CUDA::FFI.cudaGetLastError)
+    end
+    bm.report("ruby vector add") do
+      b = CooCoo::Ruby::Vector[arr]
+      10000.times do
+        b = b + b
+      end
+      #puts("Vector sum", b.inspect)
+    end
+    bm.report("nmatrix add") do
+      b = NMatrix[arr]
+      10000.times do
+        b = b + b
+      end
+      #puts("NMatrix sum", b.inspect)
+    end
+  end
+end

data/lib/coo-coo/cuda/device_buffer.rb

@@ -0,0 +1,240 @@
+require 'pathname'
+require 'ffi'
+require 'coo-coo/cuda/error'
+require 'coo-coo/cuda/host_buffer'
+
+module CooCoo
+  module CUDA
+    class DeviceBuffer < ::FFI::Struct
+      layout(:data, :pointer,
+             :size, :size_t)
+
+      def self.create(size, initial_value = 0.0)
+        FFI.new(size, initial_value.to_f)
+      end
+      
+      def self.release(ptr)
+        FFI.buffer_free(ptr)
+      rescue
+        CooCoo.debug(__method__, $!.inspect)
+      end
+
+      require 'coo-coo/cuda/device_buffer/ffi'
+      
+      def size
+        FFI.buffer_length(self)
+      end
+
+      def clone
+        self.class.
+          create(self.size).
+          set(self)
+      end
+
+      def self.[](other, length = nil)
+        if other.respond_to?(:each)
+          length ||= other.size
+        else
+          length ||= 1
+        end
+        self.create(length).set(other)
+      end
+      
+      def set(buffer)
+        case buffer
+        when self.class then FFI.set(self, buffer)
+        when Numeric then FFI.setd(self, buffer.to_f, 0, size)
+        else
+          buffer = HostBuffer[buffer]
+          FFI.setv(self, buffer.to_ptr, buffer.size)
+        end
+
+        self
+      end
+
+      def []=(index, value, length = nil)
+        index = size + index if index < 0
+        raise RangeError.new("#{index} >= #{size}") if index >= size
+        raise RangeError.new("#{index} < 0") if index < 0
+
+        if length
+          value, length = length, value
+          if value.kind_of?(self.class)
+            FFI.setn(self, index, value, length)
+          else
+            buffer = HostBuffer[value, length]
+            FFI.setvn(self, index, buffer.to_ptr, buffer.size)
+          end
+        else
+          FFI.set_element(self, index, value)
+        end
+      end
+
+      def get
+        out = HostBuffer.new(size)
+        FFI.get(self, out.to_ptr, size)
+        out
+      end
+
+      def [](index, len = nil, pad = false)
+        return super(index) if index.kind_of?(Symbol)
+        
+        index = size + index if index < 0
+        raise RangeError.new if index >= size || index < 0
+        if len
+          len = (size - index) if pad == false && (index + len) >= size
+          raise ArgumentError.new("length must be > 0") if len <= 0
+        end
+
+        if len
+          FFI.slice(self, index, len)
+        else
+          out = HostBuffer.new(1)
+          FFI.host_slice(self, out.to_ptr, index, 1)
+          out[0]
+        end
+      end
+
+      def each(&block)
+        get.each(&block)
+      end
+
+      def each_slice(n, &block)
+        return to_enum(__method__, n) unless block
+
+        (size / n.to_f).ceil.to_i.times do |i|
+          block.call(self[i * n, n, true])
+        end
+      end
+
+      def sum
+        FFI.buffer_sum(self)
+      end
+
+      def dot(w, h, other, ow = nil, oh = nil)
+        if other.kind_of?(self.class)
+          ow ||= w
+          oh ||= h
+          raise ArgumentError.new("width (#{w}) must match the other's height (#{oh})") if w != oh
+          raise ArgumentError.new("width * height != size") if size != w * h
+          raise ArgumentError.new("other's width * height != other's size (#{ow} * #{oh} != #{other.size})") if other.size != ow * oh
+          raise ArgumentError.new("other is null") if other.null?
+          raise ArgumentError.new("self is null") if null?
+          
+          FFI.dot(self, w, h, other, ow, oh)
+        else
+          b, a = coerce(other)
+          dot(w, h, b, ow, oh)
+        end
+      end
+
+      def slice_2d(width, height, x, y, out_width, out_height, initial = 0.0)
+        FFI.slice_2d(self, width, height, x, y, out_width, out_height, initial)
+      end
+
+      def set2d!(width, src, src_width, x, y)
+        case src
+        when self.class then FFI.set2d(self, width, src, src_width, x, y)
+        else
+          src = HostBuffer[src] unless src.kind_of?(HostBuffer)
+          FFI.set2dv(self, width, src.to_ptr, src_width, src.size / src_width, x, y)
+        end
+
+        self
+      end
+
+      def ==(other)
+        if other.kind_of?(self.class)
+          1 == FFI.buffer_eq(self, other)
+        else
+          return false
+        end
+      end
+
+      { :< => "lt",
+        :<= => "lte",
+        :>= => "gte",
+        :> => "gt",
+        :collect_equal? => 'eq',
+        :collect_not_equal? => 'neq'
+      }.each do |comp_op, func|
+        define_method(comp_op) do |other|
+          if other.kind_of?(self.class)
+            FFI.send("collect_#{func}", self, other)
+          elsif other.kind_of?(Numeric)
+            FFI.send("collect_#{func}d", self, other)
+          else
+            raise TypeError.new("wrong type #{other.class}")
+          end
+        end
+      end
+
+      [ :abs, :exp, :log, :log10, :log2, :sqrt,
+        :sin, :asin, :cos, :acos, :tan, :atan,
+        :sinh, :asinh, :cosh, :acosh, :tanh, :atanh,
+        :ceil, :floor, :round,
+        :collect_nan, :collect_inf
+      ].each do |f|
+        define_method(f) do
+          r = FFI.send(f, self)
+          raise NullResultError.new("NULL result") if r.null?
+          r
+        end
+      end
+      
+      def coerce(other)
+        if other.respond_to?(:each)
+          return self.class[other], self
+        else
+          return self.class.create(self.size).set(other), self
+        end
+      end
+
+      def to_a
+        get.to_a
+      end
+
+      def null?
+        super || self[:data].null?
+      end
+
+      def self.ffi_operator(op, ffi_method)
+        define_method(op) do |other|
+          if other.respond_to?(:each)
+            other = self.class[other] unless other.kind_of?(self.class)
+            raise ArgumentError.new("size mismatch: #{size} != #{other.size}") if size != other.size
+            FFI.send(ffi_method, self, other)
+          else
+            FFI.send(ffi_method.to_s + "d", self, other.to_f)
+          end
+        end
+      end
+
+      ffi_operator(:+, :add)
+      ffi_operator(:-, :sub)
+      ffi_operator(:*, :mul)
+      ffi_operator(:**, :pow)
+      ffi_operator(:/, :div)
+
+      def self.identity(w, h)
+        FFI.buffer_identity(w, h)
+      end
+
+      def diagflat
+        FFI.buffer_diagflat(self)
+      end
+
+      def min
+        FFI.buffer_min(self)
+      end
+
+      def max
+        FFI.buffer_max(self)
+      end
+
+      def minmax
+        return min, max
+      end
+    end
+  end
+end