ruby-dnn 0.1.0

data/lib/dnn/core/model.rb

@@ -0,0 +1,158 @@
+module DNN
+  class Model
+    attr_reader :layers
+    attr_reader :optimizer
+    attr_reader :batch_size
+    attr_reader :training
+  
+    def self.load(file_name)
+      Marshal.load(File.binread(file_name))
+    end
+  
+    def initialize
+      @layers = []
+      @optimizer = nil
+      @batch_size = nil
+    end
+  
+    def save(file_name)
+      File.binwrite(file_name, Marshal.dump(self))
+    end
+  
+    def <<(layer)
+      unless layer.is_a?(Layers::Layer)
+        raise DNN_TypeError.new("layer is not an instance of the DNN::Layers::Layer class.")
+      end
+      @layers << layer
+    end
+  
+    def compile(optimizer)
+      unless optimizer.is_a?(Optimizers::Optimizer)
+        raise DNN_TypeError.new("optimizer is not an instance of the DNN::Optimizers::Optimizer class.")
+      end
+      layers_check
+      @optimizer = optimizer
+      @layers.each do |layer|
+        layer.init(self)
+      end
+      layers_shape_check
+    end
+  
+    def train(x, y, epochs,
+              batch_size: 1,
+              batch_proc: nil,
+              verbose: true,
+              &epoch_proc)
+      @batch_size = batch_size
+      num_train_data = x.shape[0]
+      (1..epochs).each do |epoch|
+        puts "【 epoch #{epoch}/#{epochs} 】" if verbose
+        (num_train_data.to_f / @batch_size).ceil.times do |index|
+          x_batch, y_batch = Util.get_minibatch(x, y, @batch_size)
+          loss = train_on_batch(x_batch, y_batch, @batch_size, &batch_proc)
+          if loss.nan?
+            puts "\nloss is nan" if verbose
+            return
+          end
+          num_trained_data = (index + 1) * batch_size
+          num_trained_data = num_trained_data > num_train_data ? num_train_data : num_trained_data
+          log = "\r"
+          20.times do |i|
+            if i < num_trained_data * 20 / num_train_data
+              log << "■"
+            else
+              log << "・"
+            end
+          end
+          log << "  #{num_trained_data}/#{num_train_data} loss: #{loss}"
+          print log if verbose
+        end
+        puts "" if verbose
+        epoch_proc.call(epoch) if epoch_proc
+      end
+    end
+  
+    def train_on_batch(x, y, batch_size, &batch_proc)
+      @batch_size = batch_size
+      x, y = batch_proc.call(x, y) if batch_proc
+      forward(x, true)
+      backward(y)
+      @layers.each { |layer| layer.update if layer.respond_to?(:update) }
+      @layers[-1].loss(y)
+    end
+  
+    def test(x, y, batch_size = nil, &batch_proc)
+      @batch_size = batch_size if batch_size
+      acc = accurate(x, y, @batch_size, &batch_proc)
+      puts "accurate: #{acc}"
+      acc
+    end
+  
+    def accurate(x, y, batch_size = nil, &batch_proc)
+      @batch_size = batch_size if batch_size
+      correct = 0
+      (x.shape[0].to_f / @batch_size).ceil.times do |i|
+        x_batch = SFloat.zeros(@batch_size, *x.shape[1..-1])
+        y_batch = SFloat.zeros(@batch_size, *y.shape[1..-1])
+        @batch_size.times do |j|
+          k = i * @batch_size + j
+          break if k >= x.shape[0]
+          x_batch[j, false] = x[k, false]
+          y_batch[j, false] = y[k, false]
+        end
+        x_batch, y_batch = batch_proc.call(x_batch, y_batch) if batch_proc
+        out = forward(x_batch, false)
+        @batch_size.times do |j|
+         correct += 1 if out[j, true].max_index == y_batch[j, true].max_index
+        end
+      end
+      correct.to_f / x.shape[0]
+    end
+  
+    def predict(x)
+      forward(x, false)
+    end
+  
+    private
+  
+    def forward(x, training)
+      @training = training
+      @layers.each do |layer|
+        x = layer.forward(x)
+      end
+      x
+    end
+  
+    def backward(y)
+      dout = y
+      @layers[0..-1].reverse.each do |layer|
+        dout = layer.backward(dout)
+      end
+    end
+
+    def layers_check
+      unless @layers.first.is_a?(Layers::InputLayer)
+        raise DNN_Error.new("The first layer is not an InputLayer.")
+      end
+      unless @layers.last.is_a?(Layers::OutputLayer)
+        raise DNN_Error.new("The last layer is not an OutputLayer.")
+      end
+    end
+
+    def layers_shape_check
+      @layers.each.with_index do |layer, i|
+        if layer.is_a?(Layers::Dense)
+          prev_shape = layer.prev_layer.shape
+          if prev_shape.length != 1
+            raise DNN_SharpError.new("layer index(#{i}) Dense:  The shape of the previous layer is #{prev_shape}. The shape of the previous layer must be 1 dimensional.")
+          end
+        elsif layer.is_a?(Layers::Conv2D) || layer.is_a?(Layers::MaxPool2D)
+          prev_shape = layer.prev_layer.shape
+          if prev_shape.length != 3
+            raise DNN_SharpError.new("layer index(#{i}) Conv2D:  The shape of the previous layer is #{prev_shape}. The shape of the previous layer must be 3 dimensional.")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dnn/core/optimizers.rb

@@ -0,0 +1,113 @@
+module DNN
+  module Optimizers
+
+    #Super class of all optimizer classes.
+    class Optimizer
+      attr_accessor :learning_rate
+
+      def initialize(learning_rate)
+        @learning_rate = learning_rate
+      end
+
+      #Update layer has params.
+      def update(layer) end
+    end
+
+
+    class SGD < Optimizer
+      attr_accessor :momentum
+
+      def initialize(learning_rate = 0.01, momentum: 0)
+        super(learning_rate)
+        @momentum = momentum
+        @amounts = {}
+      end
+    
+      def update(layer)
+        amount = if @amounts[layer]
+          @amounts[layer]
+        else
+          @amounts[layer] = {}
+        end
+        layer.params.each_key do |key|
+          amount[key] = layer.grads[key] * @learning_rate
+          if @momentum > 0
+            @amounts[layer][key] ||= 0
+            amount[key] += @momentum * @amounts[layer][key]
+            @amounts[layer] = amount
+          end
+          layer.params[key] -= amount[key]
+        end
+      end
+    end
+    
+    
+    class AdaGrad    
+      def initialize(learning_rate = 0.01)
+        super(learning_rate)
+        @g = {}
+      end
+    
+      def update(layer)
+        @g[layer] ||= {}
+        layer.params.each_key do |key|
+          @g[layer][key] ||= 0
+          @g[layer][key] += layer.grads[key]**2
+          layer.params[key] -= (@learning_rate / NMath.sqrt(@g[layer][key] + 1e-7)) * layer.grads[key]
+        end
+      end
+    end
+    
+    
+    class RMSProp < Optimizer
+      attr_accessor :muse
+    
+      def initialize(learning_rate = 0.001, muse = 0.9)
+        super(learning_rate)
+        @muse = muse
+        @g = {}
+      end
+    
+      def update(layer)
+        @g[layer] ||= {}
+        layer.params.each_key do |key|
+          @g[layer][key] ||= 0
+          @g[layer][key] = @muse * @g[layer][key] + (1 - @muse) * layer.grads[key]**2
+          layer.params[key] -= (@learning_rate / NMath.sqrt(@g[layer][key] + 1e-7)) * layer.grads[key]
+        end
+      end
+    end
+
+
+    class Adam < Optimizer
+      include Numo
+      
+      attr_accessor :beta1
+      attr_accessor :beta2
+
+      def initialize(learning_rate = 0.001, beta1 = 0.9, beta2 = 0.999)
+        super(learning_rate)
+        @beta1 = beta1
+        @beta2 = beta2
+        @iter = 0
+        @m = {}
+        @v = {}
+      end
+
+      def update(layer)
+        @iter += 1
+        @m[layer] ||= {}
+        @v[layer] ||= {}
+        lr = @learning_rate * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
+        layer.params.each_key do |key|
+          @m[layer][key] ||= 0
+          @v[layer][key] ||= 0
+          @m[layer][key] += (1 - @beta1) * (layer.grads[key] - @m[layer][key])
+          @v[layer][key] += (1 - @beta2) * (layer.grads[key]**2 - @v[layer][key])
+          layer.params[key] -= lr * @m[layer][key] / NMath.sqrt(@v[layer][key] + 1e-7)
+        end
+      end
+    end
+
+  end
+end

data/lib/dnn/core/util.rb

@@ -0,0 +1,24 @@
+module DNN
+  module Util
+    #Create a mini batch for batch size.
+    def self.get_minibatch(x, y, batch_size)
+      indexes = (0...x.shape[0]).to_a.sample(batch_size)
+      [x[indexes, false], y[indexes, false]]
+    end
+
+    #Categorize labels into "num_classes" classes.
+    def self.to_categorical(y, num_classes, narray_type = nil)
+      narray_type ||= y.class
+      y2 = narray_type.zeros(y.shape[0], num_classes)
+      y.shape[0].times do |i|
+        y2[i, y[i]] = 1
+      end
+      y2
+    end
+  
+    #Perform numerical differentiation on "forward" of "layer".
+    def self.numerical_grad(x, func)
+      (func.(x + 1e-7) - func.(x)) / 1e-7
+    end
+  end
+end

data/lib/dnn/core/version.rb

@@ -0,0 +1,3 @@
+module DNN
+  VERSION = "0.1.0"
+end

data/lib/dnn/ext/cifar10/Makefile

@@ -0,0 +1,263 @@
+
+SHELL = /bin/sh
+
+# V=0 quiet, V=1 verbose.  other values don't work.
+V = 0
+Q1 = $(V:1=)
+Q = $(Q1:0=@)
+ECHO1 = $(V:1=@ :)
+ECHO = $(ECHO1:0=@ echo)
+NULLCMD = :
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /home/ootoro/.rbenv/versions/2.5.1/include/ruby-2.5.0
+hdrdir = $(topdir)
+arch_hdrdir = /home/ootoro/.rbenv/versions/2.5.1/include/ruby-2.5.0/x86_64-linux
+PATH_SEPARATOR = :
+VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
+prefix = $(DESTDIR)/home/ootoro/.rbenv/versions/2.5.1
+rubysitearchprefix = $(rubylibprefix)/$(sitearch)
+rubyarchprefix = $(rubylibprefix)/$(arch)
+rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
+exec_prefix = $(prefix)
+vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
+sitearchhdrdir = $(sitehdrdir)/$(sitearch)
+rubyarchhdrdir = $(rubyhdrdir)/$(arch)
+vendorhdrdir = $(rubyhdrdir)/vendor_ruby
+sitehdrdir = $(rubyhdrdir)/site_ruby
+rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
+vendorarchdir = $(vendorlibdir)/$(sitearch)
+vendorlibdir = $(vendordir)/$(ruby_version)
+vendordir = $(rubylibprefix)/vendor_ruby
+sitearchdir = $(sitelibdir)/$(sitearch)
+sitelibdir = $(sitedir)/$(ruby_version)
+sitedir = $(rubylibprefix)/site_ruby
+rubyarchdir = $(rubylibdir)/$(arch)
+rubylibdir = $(rubylibprefix)/$(ruby_version)
+sitearchincludedir = $(includedir)/$(sitearch)
+archincludedir = $(includedir)/$(arch)
+sitearchlibdir = $(libdir)/$(sitearch)
+archlibdir = $(libdir)/$(arch)
+ridir = $(datarootdir)/$(RI_BASE_NAME)
+mandir = $(datarootdir)/man
+localedir = $(datarootdir)/locale
+libdir = $(exec_prefix)/lib
+psdir = $(docdir)
+pdfdir = $(docdir)
+dvidir = $(docdir)
+htmldir = $(docdir)
+infodir = $(datarootdir)/info
+docdir = $(datarootdir)/doc/$(PACKAGE)
+oldincludedir = $(DESTDIR)/usr/include
+includedir = $(prefix)/include
+localstatedir = $(prefix)/var
+sharedstatedir = $(prefix)/com
+sysconfdir = $(prefix)/etc
+datadir = $(datarootdir)
+datarootdir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+sbindir = $(exec_prefix)/sbin
+bindir = $(exec_prefix)/bin
+archdir = $(rubyarchdir)
+
+
+CC = gcc
+CXX = g++
+LIBRUBY = $(LIBRUBY_A)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -Wl,-rpath,$(libdir) -L$(libdir) 
+LIBRUBYARG_STATIC = -Wl,-rpath,$(libdir) -L$(libdir) -l$(RUBY_SO_NAME)-static
+empty =
+OUTFLAG = -o $(empty)
+COUTFLAG = -o $(empty)
+CSRCFLAG = $(empty)
+
+RUBY_EXTCONF_H = 
+cflags   = $(optflags) $(debugflags) $(warnflags)
+cxxflags = $(optflags) $(debugflags) $(warnflags)
+optflags = -O3
+debugflags = -ggdb3
+warnflags = -Wall -Wextra -Wno-unused-parameter -Wno-parentheses -Wno-long-long -Wno-missing-field-initializers -Wno-tautological-compare -Wno-parentheses-equality -Wno-constant-logical-operand -Wno-self-assign -Wunused-variable -Wimplicit-int -Wpointer-arith -Wwrite-strings -Wdeclaration-after-statement -Wimplicit-function-declaration -Wdeprecated-declarations -Wno-packed-bitfield-compat -Wsuggest-attribute=noreturn -Wsuggest-attribute=format -Wno-maybe-uninitialized
+CCDLFLAGS = -fPIC
+CFLAGS   = $(CCDLFLAGS) $(cflags) $(ARCH_FLAG)
+INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
+DEFS     = 
+CPPFLAGS =  -I/home/ootoro/.rbenv/versions/2.5.1/include  $(DEFS) $(cppflags)
+CXXFLAGS = $(CCDLFLAGS) $(cxxflags) $(ARCH_FLAG)
+ldflags  = -L. -L/home/ootoro/.rbenv/versions/2.5.1/lib  -fstack-protector -rdynamic -Wl,-export-dynamic
+dldflags = -L/home/ootoro/.rbenv/versions/2.5.1/lib  -Wl,--compress-debug-sections=zlib 
+ARCH_FLAG = 
+DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
+LDSHARED = $(CC) -shared
+LDSHAREDXX = $(CXX) -shared
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
+RUBY_SO_NAME = ruby
+RUBYW_INSTALL_NAME = 
+RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
+RUBYW_BASE_NAME = rubyw
+RUBY_BASE_NAME = ruby
+
+arch = x86_64-linux
+sitearch = $(arch)
+ruby_version = 2.5.0
+ruby = $(bindir)/$(RUBY_BASE_NAME)
+RUBY = $(ruby)
+ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
+
+RM = rm -f
+RM_RF = $(RUBY) -run -e rm -- -rf
+RMDIRS = rmdir --ignore-fail-on-non-empty -p
+MAKEDIRS = /bin/mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+TOUCH = exit >
+
+#### End of system configuration section. ####
+
+preload = 
+libpath = . $(libdir)
+LIBPATH =  -L. -L$(libdir) -Wl,-rpath,$(libdir)
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+DISTCLEANDIRS = 
+
+extout = 
+extout_prefix = 
+target_prefix = 
+LOCAL_LIBS = 
+LIBS =   -lpthread -ldl -lcrypt -lm   -lc
+ORIG_SRCS = cifar10_ext.c
+SRCS = $(ORIG_SRCS) 
+OBJS = cifar10_ext.o
+HDRS = 
+LOCAL_HDRS = 
+TARGET = cifar10_ext
+TARGET_NAME = cifar10_ext
+TARGET_ENTRY = Init_$(TARGET_NAME)
+DLLIB = $(TARGET).so
+EXTSTATIC = 
+STATIC_LIB = 
+
+TIMESTAMP_DIR = .
+BINDIR        = $(bindir)
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+HDRDIR        = $(rubyhdrdir)/ruby$(target_prefix)
+ARCHHDRDIR    = $(rubyhdrdir)/$(arch)/ruby$(target_prefix)
+TARGET_SO_DIR =
+TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
+CLEANLIBS     = $(TARGET_SO) 
+CLEANOBJS     = *.o  *.bak
+
+all:    $(DLLIB)
+static: $(STATIC_LIB)
+.PHONY: all install static install-so install-rb
+.PHONY: clean clean-so clean-static clean-rb
+
+clean-static::
+clean-rb-default::
+clean-rb::
+clean-so::
+clean: clean-so clean-static clean-rb-default clean-rb
+		-$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
+
+distclean-rb-default::
+distclean-rb::
+distclean-so::
+distclean-static::
+distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
+		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
+
+realclean: distclean
+install: install-so install-rb
+
+install-so: $(DLLIB) $(TIMESTAMP_DIR)/.sitearchdir.time
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+clean-static::
+	-$(Q)$(RM) $(STATIC_LIB)
+install-rb: pre-install-rb do-install-rb install-rb-default
+install-rb-default: pre-install-rb-default do-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+do-install-rb:
+do-install-rb-default:
+pre-install-rb-default:
+	@$(NULLCMD)
+$(TIMESTAMP_DIR)/.sitearchdir.time:
+	$(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
+	$(Q) $(TOUCH) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
+
+.cc.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cc.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.mm.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.mm.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cxx.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cxx.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.cpp.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.cpp.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.c.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.c.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+.m.o:
+	$(ECHO) compiling $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
+
+.m.S:
+	$(ECHO) translating $(<)
+	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
+
+$(TARGET_SO): $(OBJS) Makefile
+	$(ECHO) linking shared-object $(DLLIB)
+	-$(Q)$(RM) $(@)
+	$(Q) $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): $(HDRS) $(ruby_headers)