red-datasets 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d071f7be3f241f1fb4327e63666c35879488f47c63e9844f8e86d099e385d79
-  data.tar.gz: 7e688dfc0ccc9d0ca8bc0070eef71dee1f3e7732e8887d443b37a577467dbf75
+  metadata.gz: 56c3132aec99f8dab763d68d0b731df0e5b2c97ea9f88a847021210fda94b9ac
+  data.tar.gz: 20e11d381718b801dac9c155385837b770c5067646ac235ade9c0f06f4d23cb8
 SHA512:
-  metadata.gz: f71bf4fbb25332709d4ef4c8ddc1121781ecac05097551d336091c875a5c885fd20bbba658b41085faf3d1433c29ece43d458c104c300f6eeaa0d8088eae6377
-  data.tar.gz: cca27dc33ed60f0093bcf940068590df4fb0848b6f40ba265b9bf53316c888267d8c561248fdc5c755cbf5c93ded2a5068d6f8369e5ec6951bee367c552d8677
+  metadata.gz: 1c3ccf7b09e6064cacc47c04dd972ef4ba9318278630446ae60911412e9dc5dced5c988423b78a1c0ae68d71ede6136ebd2ee9c86f4247fd56bbf940f7997f7e
+  data.tar.gz: bebe49f31022dfeab1eb7626ef1eaf722d5dedf373dab89ba79e3e2afb4849ace50b5098688e4d0480f22719a758a086c288c600cdcb358099a7aa9d1e60eab1

data/README.md

@@ -37,6 +37,44 @@ iris.each do |record|
 end
 ```
 
+Here is an example to access CIFAR dataset by `#each`:
+
+**CIFAR-10**
+
+```ruby
+require "datasets"
+
+cifar = Datasets::CIFAR.new(n_classes: 10, type: :train)
+cifar.metadata
+#=> #<struct Datasets::Metadata name="CIFAR-10", url="https://www.cs.toronto.edu/~kriz/cifar.html", licenses=nil, description="CIFAR-10 is 32x32 image dataset">licenses=nil, description="CIFAR-10 is 32x32 image datasets">
+cifar.each do |record|
+    p record.pixels
+    # => [59, 43, 50, 68, 98, 119, 139, 145, 149, 143, .....]
+    p record.label
+    # => 6
+  end
+end
+```
+
+**CIFAR-100**
+
+```ruby
+require "datasets"
+
+cifar = Datasets::CIFAR.new(n_classes: 100, type: :test)
+cifar.metadata
+#=> #<struct Datasets::Metadata name="CIFAR-100", url="https://www.cs.toronto.edu/~kriz/cifar.html", licenses=nil, description="CIFAR-100 is 32x32 image dataset">
+cifar.each do |record|
+    p record.pixels
+    #=> [199, 196, 195, 195, 196, 197, 198, 198, 199, .....]
+    p record.coarse_label
+    #=> 10
+    p record.fine_label
+    #=> 49
+  end
+end
+```
+
 ## License
 
 The MIT license. See `LICENSE.txt` for details.

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # News
 
+## 0.0.4 - 2018-05-03
+
+### Improvements
+
+  * `Datasets::Dataset`: Made enumerable.
+
+  * `Datasets::CIFAR`: Added the CIFAR dataset.
+    [GitHub#7][GitHub#8][GitHub#9][GitHub#10]
+    [Patch by Yusaku Hatanaka]
+
+### Thanks
+
+  * Yusaku Hatanaka
+
 ## 0.0.3 - 2018-03-27
 
 ### Improvements

data/lib/datasets/cifar.rb

@@ -0,0 +1,126 @@
+require "rubygems/package"
+require "zlib"
+
+require_relative "dataset"
+
+module Datasets
+  class CIFAR < Dataset
+    class Record10 < Struct.new(:data, :label)
+      def pixels
+        data.unpack("C*")
+      end
+    end
+
+    class Record100 < Struct.new(:data, :coarse_label, :fine_label)
+      def pixels
+        data.unpack("C*")
+      end
+    end
+
+    def initialize(n_classes: 10, type: :train)
+      unless [10, 100].include?(n_classes)
+        raise 'Please set n_classes 10 or 100'
+      end
+      unless [:train, :test].include?(type)
+        raise 'Please set type :train or :test'
+      end
+
+      super()
+
+      @metadata.name = "CIFAR-#{n_classes}"
+      @metadata.url = "https://www.cs.toronto.edu/~kriz/cifar.html"
+      @metadata.description = "CIFAR-#{n_classes} is 32x32 image dataset"
+
+      @n_classes = n_classes
+      @type = type
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "cifar-#{@n_classes}.tar.gz"
+      unless data_path.exist?
+        data_url = "https://www.cs.toronto.edu/~kriz/cifar-#{@n_classes}-binary.tar.gz"
+        download(data_path, data_url)
+      end
+
+      parse_data(data_path, &block)
+    end
+
+    private
+
+    def parse_data(data_path, &block)
+      open_tar(data_path) do |tar|
+        target_file_names.each do |target_file_name|
+          tar.seek(target_file_name) do |entry|
+            parse_entry(entry, &block)
+          end
+        end
+      end
+    end
+
+    def target_file_names
+      case @n_classes
+      when 10
+        prefix = 'cifar-10-batches-bin'
+        case @type
+        when :train
+          [
+            "#{prefix}/data_batch_1.bin",
+            "#{prefix}/data_batch_2.bin",
+            "#{prefix}/data_batch_3.bin",
+            "#{prefix}/data_batch_4.bin",
+            "#{prefix}/data_batch_5.bin",
+          ]
+        when :test
+          [
+            "#{prefix}/test_batch.bin"
+          ]
+        end
+      when 100
+        prefix = "cifar-100-binary"
+        case @type
+        when :train
+          [
+            "#{prefix}/train.bin",
+          ]
+        when :test
+          [
+            "#{prefix}/test.bin",
+          ]
+        end
+      end
+    end
+
+    def parse_entry(entry)
+      case @n_classes
+      when 10
+        loop do
+          label = entry.read(1)
+          break if label.nil?
+          label = label.unpack("C")[0]
+          data = entry.read(3072)
+          yield Record10.new(data, label)
+        end
+      when 100
+        loop do
+          coarse_label = entry.read(1)
+          break if coarse_label.nil?
+          coarse_label = coarse_label.unpack("C")[0]
+          fine_label = entry.read(1).unpack("C")[0]
+          data = entry.read(3072)
+          yield Record100.new(data, coarse_label, fine_label)
+        end
+      end
+    end
+
+    def open_tar(data_path)
+      Zlib::GzipReader.open(data_path) do |f|
+        Gem::Package::TarReader.new(f) do |tar|
+          yield(tar)
+        end
+      end
+    end
+  end
+end
+

data/lib/datasets/dataset.rb

@@ -6,6 +6,8 @@ require_relative "table"
 
 module Datasets
   class Dataset
+    include Enumerable
+
     attr_reader :metadata
     def initialize
       @metadata = Metadata.new

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/lib/datasets.rb

@@ -1,4 +1,5 @@
 require "datasets/version"
 
+require "datasets/cifar"
 require "datasets/iris"
 require "datasets/wikipedia"

data/test/test-cifar.rb

@@ -0,0 +1,209 @@
+class CIFARTest < Test::Unit::TestCase
+  include Helper::Sandbox
+
+  def setup_raw_data(data)
+    setup_sandbox
+
+    def @dataset.cache_dir_path
+      @cache_dir_path
+    end
+    def @dataset.cache_dir_path=(path)
+      @cache_dir_path = path
+    end
+    @dataset.cache_dir_path = @tmp_dir
+
+    def @dataset.data=(data)
+      @data = data
+    end
+    @dataset.data = data
+
+    def @dataset.download(output_path, url)
+      Zlib::GzipWriter.open(output_path) do |gz|
+        Gem::Package::TarWriter.new(gz) do |tar|
+          @data.each do |path, content|
+            if content == :directory
+              tar.mkdir(path, 0755)
+            else
+              tar.add_file_simple(path, 0644, content.bytesize) do |file|
+                file.write(content)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def teardown
+    teardown_sandbox
+  end
+
+  sub_test_case("cifar-10") do
+    def create_data(label, pixel)
+      [label].pack("C") + ([pixel] * 3072).pack("C*")
+    end
+
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::CIFAR.new(n_classes: 10, type: :train)
+        directory = "cifar-10-batches-bin"
+        setup_raw_data(directory => :directory,
+                       "#{directory}/data_batch_1.bin" => create_data(1, 10),
+                       "#{directory}/data_batch_2.bin" => create_data(2, 20),
+                       "#{directory}/data_batch_3.bin" => create_data(3, 30),
+                       "#{directory}/data_batch_4.bin" => create_data(4, 40),
+                       "#{directory}/data_batch_5.bin" => create_data(5, 50),
+                       "#{directory}/data_batch_6.bin" => create_data(6, 60))
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :label => record.label,
+            :data => record.data,
+            :pixels => record.pixels
+          }
+        end
+        assert_equal([
+                       {
+                         :label => 1,
+                         :data => ([10] * 3072).pack("C*"),
+                         :pixels => [10] * 3072
+                       },
+                       {
+                         :label => 2,
+                         :data => ([20] * 3072).pack("C*"),
+                         :pixels => [20] * 3072
+                       },
+                       {
+                         :label => 3,
+                         :data => ([30] * 3072).pack("C*"),
+                         :pixels => [30] * 3072
+                       },
+                       {
+                         :label => 4,
+                         :data => ([40] * 3072).pack("C*"),
+                         :pixels => [40] * 3072
+                       },
+                       {
+                         :label => 5,
+                         :data => ([50] * 3072).pack("C*"),
+                         :pixels => [50] * 3072
+                       },
+                     ],
+                     raw_dataset)
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::CIFAR.new(n_classes: 10, type: :test)
+        directory = "cifar-10-batches-bin"
+        data = create_data(1, 100) + create_data(2, 200)
+        setup_raw_data(directory => :directory,
+                       "#{directory}/test_batch.bin" => data)
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :label => record.label,
+            :data => record.data,
+            :pixels => record.pixels,
+          }
+        end
+        assert_equal([
+                       {
+                         :label => 1,
+                         :data => ([100] * 3072).pack("C*"),
+                         :pixels => [100] * 3072
+                       },
+                       {
+                         :label => 2,
+                         :data => ([200] * 3072).pack("C*"),
+                         :pixels => [200] * 3072
+                       },
+                     ],
+                     raw_dataset)
+      end
+    end
+  end
+
+  sub_test_case("cifar-100") do
+    def create_data(coarse_label, fine_label, pixel)
+      [coarse_label, fine_label].pack("C*") + ([pixel] * 3072).pack("C*")
+    end
+
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::CIFAR.new(n_classes: 100, type: :train)
+        directory = "cifar-100-binary"
+        data = create_data(1, 11, 10) + create_data(2, 22, 20)
+        setup_raw_data(directory => :directory,
+                       "#{directory}/train.bin" => data)
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :coarse_label => record.coarse_label,
+            :fine_label => record.fine_label,
+            :data => record.data,
+            :pixels => record.pixels,
+          }
+        end
+        assert_equal([
+                       {
+                         :coarse_label => 1,
+                         :fine_label => 11,
+                         :data => ([10] * 3072).pack("C*"),
+                         :pixels => [10] * 3072
+                       },
+                       {
+                         :coarse_label => 2,
+                         :fine_label => 22,
+                         :data => ([20] * 3072).pack("C*"),
+                         :pixels => [20] * 3072
+                       },
+                     ],
+                     raw_dataset)
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::CIFAR.new(n_classes: 100, type: :test)
+        directory = "cifar-100-binary"
+        data = create_data(1, 11, 100) + create_data(6, 66, 200)
+        setup_raw_data(directory => :directory,
+                       "#{directory}/test.bin" => data)
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :coarse_label => record.coarse_label,
+            :fine_label => record.fine_label,
+            :data => record.data,
+            :pixels => record.pixels,
+          }
+        end
+        assert_equal([
+                       {
+                         :coarse_label => 1,
+                         :fine_label => 11,
+                         :data => ([100] * 3072).pack("C*"),
+                         :pixels => [100] * 3072
+                       },
+                       {
+                         :coarse_label => 6,
+                         :fine_label => 66,
+                         :data => ([200] * 3072).pack("C*"),
+                         :pixels => [200] * 3072
+                       },
+                     ],
+                     raw_dataset)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - tomisuker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-27 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -99,6 +99,7 @@ files:
 - Rakefile
 - doc/text/news.md
 - lib/datasets.rb
+- lib/datasets/cifar.rb
 - lib/datasets/dataset.rb
 - lib/datasets/downloader.rb
 - lib/datasets/iris.rb
@@ -109,6 +110,7 @@ files:
 - red-datasets.gemspec
 - test/helper.rb
 - test/run-test.rb
+- test/test-cifar.rb
 - test/test-iris.rb
 - test/test-table.rb
 - test/test-wikipedia.rb
@@ -142,3 +144,4 @@ test_files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-table.rb
+- test/test-cifar.rb