ruby-dnn 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f0d01c2a4696fdf876c6f247a7ea7e41bd57f4d42d2e0065a3703c4faf5a7cc
-  data.tar.gz: 29bcebf1ae90f3b6f72dd391fbba9118162dc49f490b801d1e634f736cc580a4
+  metadata.gz: 8156ccf7edaf425d09a1463bb1c9280a1eeddaa67448323a19d301d451a78a0f
+  data.tar.gz: 851e5aa8c87fd60ee9d5296f995c9b884a1bcaed823ab9e5d034f0b7b52d740b
 SHA512:
-  metadata.gz: b7b01228b299fbf42ec74f0d4678e6ffd62775a606ecadba3253ce76fd62166699851450d359f13022b01023219b8f710ef75c75af6c1096591df180a49edd62
-  data.tar.gz: 2eb11ae75136aa6be186d81c1fe8c04d585167ce31feccdd26406d58cafd6f333080c871723ad533e9f3ad0fb677c08180ee9925e99040f093f90a08a7f24631
+  metadata.gz: 520b9dc2ff534d36d86af82750b12a1772d29b487cfc18e02425f14c006e9fc1a15c525a0f1cfbe1ce541d2c7cac0cc7649a0542f3b9448634daf9869471f5d3
+  data.tar.gz: 2ffe038bc5e5b69c61f32f99e9b21df8b2321b32148c6226d85e9aa82cedd8bade8bcbdab66c983a827a923ab3d3e449bb03cedb4c52f9b34fcbd1a732220e43

data/lib/dnn/ext/cifar10_loader/cifar10_loader.c

@@ -0,0 +1,43 @@
+#include <ruby.h>
+#include <stdlib.h>
+
+#define CIFAR10_WIDTH 32
+#define CIFAR10_HEIGHT 32
+#define CIFAR10_CHANNEL 3
+
+VALUE cifar10_load_binary(VALUE self, VALUE rb_bin, VALUE rb_num_datas) {
+  char* bin = StringValuePtr(rb_bin);
+  int num_datas = FIX2INT(rb_num_datas);
+  VALUE rb_x_bin;
+  VALUE rb_y_bin;
+  int i;
+  int j = 0;
+  int k = 0;
+  int size = CIFAR10_WIDTH * CIFAR10_HEIGHT * CIFAR10_CHANNEL;
+  int x_bin_size = num_datas * size;
+  int y_bin_size = num_datas;
+  char* x_bin;
+  char* y_bin;
+
+  x_bin = (char*)malloc(x_bin_size);
+  y_bin = (char*)malloc(y_bin_size);
+  for (i = 0; i < num_datas; i++) {
+    y_bin[i] = bin[j];
+    j++;
+    memcpy(&x_bin[k], &bin[j], size);
+    j += size;
+    k += size;
+  }
+  rb_x_bin = rb_str_new(x_bin, x_bin_size);
+  rb_y_bin = rb_str_new(y_bin, y_bin_size);
+  free(x_bin);
+  free(y_bin);
+  return rb_ary_new3(2, rb_x_bin, rb_y_bin);
+}
+
+void Init_cifar10_loader() {
+  VALUE rb_dnn = rb_define_module("DNN");
+  VALUE rb_cifar10 = rb_define_module_under(rb_dnn, "CIFAR10");
+
+  rb_define_singleton_method(rb_cifar10, "load_binary", cifar10_load_binary, 2);
+}

data/lib/dnn/ext/cifar10_loader/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile("cifar10_loader")

data/lib/dnn/ext/rb_stb_image/rb_stb_image.c

@@ -1,5 +1,4 @@
 #include <ruby.h>
-#include "numo/narray.h"
 
 #define STB_IMAGE_IMPLEMENTATION
 #define STB_IMAGE_WRITE_IMPLEMENTATION
@@ -12,79 +11,62 @@ VALUE rb_stbi_load(VALUE self, VALUE rb_filename, VALUE rb_req_comp) {
   char* filename = StringValuePtr(rb_filename);
   int x, y, n;
   int req_comp = FIX2INT(rb_req_comp);
-  unsigned char* pixels;
-  narray_data_t* na_data;
-  char script[64];
+  unsigned char* data;
   int ch;
   VALUE rb_x, rb_y, rb_n;
-  VALUE rb_pixels;
+  VALUE rb_data;
 
-  pixels = stbi_load(filename, &x, &y, &n, req_comp);
+  data = stbi_load(filename, &x, &y, &n, req_comp);
   rb_x = INT2FIX(x);
   rb_y = INT2FIX(y);
   rb_n = INT2FIX(n);
   ch = req_comp == 0 ? n : req_comp;
-  sprintf(script, "Numo::UInt8.zeros(%d, %d, %d)", y, x, ch);
-  rb_pixels = rb_eval_string(&script[0]);
-  na_data = RNARRAY_DATA(rb_pixels);
-  memcpy(na_data->ptr, pixels, na_data->base.size);
-  stbi_image_free(pixels);
-  return rb_ary_new3(4, rb_pixels, rb_x, rb_y, rb_n);
+  rb_data = rb_str_new((char*)data, y * x * ch);
+  stbi_image_free(data);
+  return rb_ary_new3(4, rb_data, rb_x, rb_y, rb_n);
 }
 
 //STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void  *data, int stride_in_bytes);
-VALUE rb_stbi_write_png(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_pixels, VALUE rb_stride_in_bytes) {
+VALUE rb_stbi_write_png(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_data, VALUE rb_stride_in_bytes) {
   char* filename = StringValuePtr(rb_filename);
   int w = FIX2INT(rb_w);
   int h = FIX2INT(rb_h);
   int comp = FIX2INT(rb_comp);
-  unsigned char* pixels;
+  unsigned char* data = (unsigned char*)StringValuePtr(rb_data);
   int stride_in_bytes = FIX2INT(rb_stride_in_bytes);
-  narray_data_t* na_data;
   int result;
 
-  na_data = RNARRAY_DATA(rb_pixels);
-  pixels = (unsigned char*)malloc(na_data->base.size);
-  memcpy(pixels, na_data->ptr, na_data->base.size);
-  result = stbi_write_png(filename, w, h, comp, pixels, stride_in_bytes);
-  stbi_image_free(pixels);
+  result = stbi_write_png(filename, w, h, comp, data, stride_in_bytes);
+  stbi_image_free(data);
   return INT2FIX(result);
 }
 
 //STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void  *data);
-VALUE rb_stbi_write_bmp(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_pixels) {
+VALUE rb_stbi_write_bmp(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_data) {
   char* filename = StringValuePtr(rb_filename);
   int w = FIX2INT(rb_w);
   int h = FIX2INT(rb_h);
   int comp = FIX2INT(rb_comp);
-  unsigned char* pixels;
-  narray_data_t* na_data;
+  unsigned char* data = (unsigned char*)StringValuePtr(rb_data);
   int result;
 
-  na_data = RNARRAY_DATA(rb_pixels);
-  pixels = (unsigned char*)malloc(na_data->base.size);
-  memcpy(pixels, na_data->ptr, na_data->base.size);
-  result = stbi_write_bmp(filename, w, h, comp, pixels);
-  stbi_image_free(pixels);
+  result = stbi_write_bmp(filename, w, h, comp, data);
+  stbi_image_free(data);
   return INT2FIX(result);
 }
 
 //STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void  *data, int quality);
-VALUE rb_stbi_write_jpg(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_pixels, VALUE rb_quality) {
+VALUE rb_stbi_write_jpg(VALUE self, VALUE rb_filename, VALUE rb_w, VALUE rb_h, VALUE rb_comp, VALUE rb_data, VALUE rb_quality) {
   char* filename = StringValuePtr(rb_filename);
   int w = FIX2INT(rb_w);
   int h = FIX2INT(rb_h);
   int comp = FIX2INT(rb_comp);
-  unsigned char* pixels;
+  unsigned char* data = (unsigned char*)StringValuePtr(rb_data);
   int quality = FIX2INT(rb_quality);
-  narray_data_t* na_data;
   int result;
 
-  na_data = RNARRAY_DATA(rb_pixels);
-  pixels = (unsigned char*)malloc(na_data->base.size);
-  memcpy(pixels, na_data->ptr, na_data->base.size);
-  result = stbi_write_jpg(filename, w, h, comp, pixels, quality);
-  stbi_image_free(pixels);
+  result = stbi_write_jpg(filename, w, h, comp, data, quality);
+  stbi_image_free(data);
   return INT2FIX(result);
 }
 

data/lib/dnn/lib/cifar10.rb

@@ -1,5 +1,5 @@
 require "dnn"
-require "dnn/ext/dataset_loader/dataset_loader"
+require "dnn/ext/cifar10_loader/cifar10_loader"
 require "open-uri"
 require "zlib"
 require "archive/tar/minitar"
@@ -13,7 +13,7 @@ module DNN
 
     class DNN_CIFAR10_DownloadError < DNN_Error; end
 
-    private_class_method :_cifar10_load
+    private_class_method :load_binary
 
     def self.downloads
       return if Dir.exist?(__dir__ + "/" + CIFAR10_DIR)
@@ -42,8 +42,9 @@ module DNN
         raise DNN_CIFAR10_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
         bin << File.binread(fname)
       end
-      x_train, y_train = _cifar10_load(bin, 50000)
-      x_train = x_train.transpose(0, 2, 3, 1).clone
+      x_bin, y_bin = load_binary(bin, 50000)
+      x_train = Numo::UInt8.from_binary(x_bin).reshape(50000, 3, 32, 32).transpose(0, 2, 3, 1).clone
+      y_train = Numo::UInt8.from_binary(y_bin)
       [x_train, y_train]
     end
 
@@ -52,8 +53,9 @@ module DNN
       fname = __dir__ + "/#{CIFAR10_DIR}/test_batch.bin"
       raise DNN_CIFAR10_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
       bin = File.binread(fname)
-      x_test, y_test = _cifar10_load(bin, 10000)
-      x_test = x_test.transpose(0, 2, 3, 1).clone
+      x_bin, y_bin = load_binary(bin, 10000)
+      x_test = Numo::UInt8.from_binary(x_bin).reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).clone
+      y_test = Numo::UInt8.from_binary(y_bin)
       [x_test, y_test]
     end
   end

data/lib/dnn/lib/image_io.rb

@@ -5,26 +5,28 @@ module DNN
   module ImageIO
     def self.read(file_name)
       raise ImageIO::ReadError.new("#{file_name} is not found.") unless File.exist?(file_name)
-      img, = Stb.stbi_load(file_name, 3)
-      img
+      bin, w, h, n = Stb.stbi_load(file_name, 3)
+      img = Numo::UInt8.from_binary(bin)
+      img.reshape(h, w, 3)
     end
 
     def self.write(file_name, img, quality: 100)
-      img = img.clone
       if img.shape.length == 2
         img = Numo::UInt8[img, img, img].transpose(1, 2, 0).clone
       elsif img.shape[2] == 1
         img = img.shape(img.shape[0], img.shape[1])
         img = Numo::UInt8[img, img, img].transpose(1, 2, 0).clone
       end
+      h, w, ch = img.shape
+      bin = img.to_binary
       case file_name
       when /\.png$/
-        stride_in_bytes = img.shape[0] * img.shape[2]
-        Stb.stbi_write_png(file_name, *img.shape, img, stride_in_bytes)
+        stride_in_bytes = w * ch
+        Stb.stbi_write_png(file_name, w, h, ch, bin, stride_in_bytes)
       when /\.bmp$/
-        Stb.stbi_write_bmp(file_name, *img.shape, img)
+        Stb.stbi_write_bmp(file_name, w, h, ch, bin)
       when /\.jpg$/
-        Stb.stbi_write_jpg(file_name, *img.shape, img, quality)
+        Stb.stbi_write_jpg(file_name, w, h, ch, bin, quality)
       end
     rescue => ex
       raise ImageIO::WriteError.new(ex.message)

data/lib/dnn/lib/mnist.rb

@@ -73,7 +73,8 @@ module DNN
       Zlib::GzipReader.open(file_name) do |f|
         magic, num_images = f.read(8).unpack("N2")
         rows, cols = f.read(8).unpack("N2")
-        images = _mnist_load_images(f.read, num_images, cols, rows)
+        images = Numo::UInt8.from_binary(f.read)
+        images = images.reshape(num_images, cols, rows)
       end
       images
     end
@@ -82,7 +83,7 @@ module DNN
       labels = nil
       Zlib::GzipReader.open(file_name) do |f|
         magic, num_labels = f.read(8).unpack("N2")
-        labels = _mnist_load_labels(f.read, num_labels)
+        labels = Numo::UInt8.from_binary(f.read)
       end
       labels
     end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/ruby-dnn.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.description   = %q{ruby-dnn is a ruby deep learning library.}
   spec.homepage      = "https://github.com/unagiootoro/ruby-dnn.git"
   spec.license       = "MIT"
-  spec.extensions    = ["lib/dnn/ext/dataset_loader/extconf.rb", "lib/dnn/ext/rb_stb_image/extconf.rb"]
+  spec.extensions    = ["lib/dnn/ext/cifar10_loader/extconf.rb", "lib/dnn/ext/rb_stb_image/extconf.rb"]
 
   spec.add_dependency "numo-narray"
   spec.add_dependency "archive-tar-minitar"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-21 00:00:00.000000000 Z
+date: 2018-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -85,7 +85,7 @@ email:
 - ootoro838861@outlook.jp
 executables: []
 extensions:
-- lib/dnn/ext/dataset_loader/extconf.rb
+- lib/dnn/ext/cifar10_loader/extconf.rb
 - lib/dnn/ext/rb_stb_image/extconf.rb
 extra_rdoc_files: []
 files:
@@ -112,67 +112,9 @@ files:
 - lib/dnn/core/model.rb
 - lib/dnn/core/optimizers.rb
 - lib/dnn/core/util.rb
-- lib/dnn/ext/dataset_loader/dataset_loader.c
-- lib/dnn/ext/dataset_loader/extconf.rb
-- lib/dnn/ext/dataset_loader/numo/compat.h
-- lib/dnn/ext/dataset_loader/numo/extconf.h
-- lib/dnn/ext/dataset_loader/numo/intern.h
-- lib/dnn/ext/dataset_loader/numo/narray.h
-- lib/dnn/ext/dataset_loader/numo/ndloop.h
-- lib/dnn/ext/dataset_loader/numo/template.h
-- lib/dnn/ext/dataset_loader/numo/types/bit.h
-- lib/dnn/ext/dataset_loader/numo/types/complex.h
-- lib/dnn/ext/dataset_loader/numo/types/complex_macro.h
-- lib/dnn/ext/dataset_loader/numo/types/dcomplex.h
-- lib/dnn/ext/dataset_loader/numo/types/dfloat.h
-- lib/dnn/ext/dataset_loader/numo/types/float_def.h
-- lib/dnn/ext/dataset_loader/numo/types/float_macro.h
-- lib/dnn/ext/dataset_loader/numo/types/int16.h
-- lib/dnn/ext/dataset_loader/numo/types/int32.h
-- lib/dnn/ext/dataset_loader/numo/types/int64.h
-- lib/dnn/ext/dataset_loader/numo/types/int8.h
-- lib/dnn/ext/dataset_loader/numo/types/int_macro.h
-- lib/dnn/ext/dataset_loader/numo/types/real_accum.h
-- lib/dnn/ext/dataset_loader/numo/types/robj_macro.h
-- lib/dnn/ext/dataset_loader/numo/types/robject.h
-- lib/dnn/ext/dataset_loader/numo/types/scomplex.h
-- lib/dnn/ext/dataset_loader/numo/types/sfloat.h
-- lib/dnn/ext/dataset_loader/numo/types/uint16.h
-- lib/dnn/ext/dataset_loader/numo/types/uint32.h
-- lib/dnn/ext/dataset_loader/numo/types/uint64.h
-- lib/dnn/ext/dataset_loader/numo/types/uint8.h
-- lib/dnn/ext/dataset_loader/numo/types/uint_macro.h
-- lib/dnn/ext/dataset_loader/numo/types/xint_macro.h
+- lib/dnn/ext/cifar10_loader/cifar10_loader.c
+- lib/dnn/ext/cifar10_loader/extconf.rb
 - lib/dnn/ext/rb_stb_image/extconf.rb
-- lib/dnn/ext/rb_stb_image/numo/compat.h
-- lib/dnn/ext/rb_stb_image/numo/extconf.h
-- lib/dnn/ext/rb_stb_image/numo/intern.h
-- lib/dnn/ext/rb_stb_image/numo/narray.h
-- lib/dnn/ext/rb_stb_image/numo/ndloop.h
-- lib/dnn/ext/rb_stb_image/numo/template.h
-- lib/dnn/ext/rb_stb_image/numo/types/bit.h
-- lib/dnn/ext/rb_stb_image/numo/types/complex.h
-- lib/dnn/ext/rb_stb_image/numo/types/complex_macro.h
-- lib/dnn/ext/rb_stb_image/numo/types/dcomplex.h
-- lib/dnn/ext/rb_stb_image/numo/types/dfloat.h
-- lib/dnn/ext/rb_stb_image/numo/types/float_def.h
-- lib/dnn/ext/rb_stb_image/numo/types/float_macro.h
-- lib/dnn/ext/rb_stb_image/numo/types/int16.h
-- lib/dnn/ext/rb_stb_image/numo/types/int32.h
-- lib/dnn/ext/rb_stb_image/numo/types/int64.h
-- lib/dnn/ext/rb_stb_image/numo/types/int8.h
-- lib/dnn/ext/rb_stb_image/numo/types/int_macro.h
-- lib/dnn/ext/rb_stb_image/numo/types/real_accum.h
-- lib/dnn/ext/rb_stb_image/numo/types/robj_macro.h
-- lib/dnn/ext/rb_stb_image/numo/types/robject.h
-- lib/dnn/ext/rb_stb_image/numo/types/scomplex.h
-- lib/dnn/ext/rb_stb_image/numo/types/sfloat.h
-- lib/dnn/ext/rb_stb_image/numo/types/uint16.h
-- lib/dnn/ext/rb_stb_image/numo/types/uint32.h
-- lib/dnn/ext/rb_stb_image/numo/types/uint64.h
-- lib/dnn/ext/rb_stb_image/numo/types/uint8.h
-- lib/dnn/ext/rb_stb_image/numo/types/uint_macro.h
-- lib/dnn/ext/rb_stb_image/numo/types/xint_macro.h
 - lib/dnn/ext/rb_stb_image/rb_stb_image.c
 - lib/dnn/ext/rb_stb_image/stb_image.h
 - lib/dnn/ext/rb_stb_image/stb_image_write.h

data/lib/dnn/ext/dataset_loader/dataset_loader.c

@@ -1,90 +0,0 @@
-#include <ruby.h>
-#include <stdlib.h>
-#include "numo/narray.h"
-
-#define CIFAR10_WIDTH 32
-#define CIFAR10_HEIGHT 32
-#define CIFAR10_CHANNEL 3
-
-VALUE mnist_load_images(VALUE self, VALUE rb_bin, VALUE rb_num_images, VALUE rb_cols, VALUE rb_rows) {
-  char* bin = StringValuePtr(rb_bin);
-  int num_images = FIX2INT(rb_num_images);
-  int cols = FIX2INT(rb_cols);
-  int rows = FIX2INT(rb_rows);
-  int i;
-  int j;
-  char script[64];
-  VALUE rb_na;
-  narray_data_t* na_data;
-
-  sprintf(script, "Numo::UInt8.zeros(%d, %d, %d)", num_images, cols, rows);
-  rb_na = rb_eval_string(&script[0]);
-  na_data = RNARRAY_DATA(rb_na);
-
-  for (i = 0; i < num_images; i++) {
-    j = i * cols * rows;
-    memcpy(&na_data->ptr[j], &bin[j], cols * rows);
-  }
-  return rb_na;
-}
-
-VALUE mnist_load_labels(VALUE self, VALUE rb_bin, VALUE rb_num_labels) {
-  char* bin = StringValuePtr(rb_bin);
-  int num_labels = FIX2INT(rb_num_labels);
-  char script[64];
-  VALUE rb_na;
-  narray_data_t* na_data;
-
-  sprintf(script, "Numo::UInt8.zeros(%d)", num_labels);
-  rb_na = rb_eval_string(&script[0]);
-  na_data = RNARRAY_DATA(rb_na);
-
-  memcpy(na_data->ptr, bin, num_labels);
-  return rb_na;
-}
-
-VALUE cifar10_load(VALUE self, VALUE rb_bin, VALUE rb_num_datas) {
-  char* bin = StringValuePtr(rb_bin);
-  int num_datas = FIX2INT(rb_num_datas);
-  char script[64];
-  VALUE rb_na_x;
-  VALUE rb_na_y;
-  VALUE rb_xy;
-  narray_data_t* na_data_x;
-  narray_data_t* na_data_y;
-  int i;
-  int j = 0;
-  int k = 0;
-  int size = CIFAR10_WIDTH * CIFAR10_HEIGHT * CIFAR10_CHANNEL;
-
-  sprintf(script, "Numo::UInt8.zeros(%d, %d, %d, %d)", num_datas, CIFAR10_CHANNEL, CIFAR10_WIDTH, CIFAR10_HEIGHT);
-  rb_na_x = rb_eval_string(&script[0]);
-  na_data_x = RNARRAY_DATA(rb_na_x);
-  for(i = 0; i < 64; i++) {
-    script[i] = 0;
-  }
-  sprintf(script, "Numo::UInt8.zeros(%d)", num_datas);
-  rb_na_y = rb_eval_string(&script[0]);
-  na_data_y = RNARRAY_DATA(rb_na_y);
-
-  for (i = 0; i < num_datas; i++) {
-    na_data_y->ptr[i] = bin[j];
-    j++;
-    memcpy(&na_data_x->ptr[k], &bin[j], size);
-    j += size;
-    k += size;
-  }
-
-  rb_xy = rb_ary_new3(2, rb_na_x, rb_na_y);
-  return rb_xy;
-}
-
-void Init_dataset_loader() {
-  VALUE rb_dnn = rb_define_module("DNN");
-  VALUE rb_mnist = rb_define_module_under(rb_dnn, "MNIST");
-  VALUE rb_cifar10 = rb_define_module_under(rb_dnn, "CIFAR10");
-
-  rb_define_singleton_method(rb_mnist, "_mnist_load_images", mnist_load_images, 4);
-  rb_define_singleton_method(rb_mnist, "_mnist_load_labels", mnist_load_labels, 2); 
-  rb_define_singleton_method(rb_cifar10, "_cifar10_load", cifar10_load, 2);
-}