red-datasets 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b52a97ab0ea10ea2d5ca5b873adab2e57aa8f8a71093e9e94c13da55bc8f774
-  data.tar.gz: 5f5525fc4cda3a9a57c08f6855fedd4da3c4be8b72fb7b35cee0e99133492c1f
+  metadata.gz: 3b96f5bf8fb7d8d7280451086dda394b65c42023b15ae077167e2d320c4361c1
+  data.tar.gz: 96f7936d62d70749f92d3bdd1d7ef2d79cfff3091e7dae8221d6a0537dbd6d7b
 SHA512:
-  metadata.gz: ae45fd3d9a6128ddca38c2b37a3a0c8fa89c831bdb1e14c1fdda2183be29385f74df6259ec4ae36934b8c0db57c9d2434208beeeaed854ea254ec6c327f21d64
-  data.tar.gz: 45209c6e106d78d008e73ab4414a933bc8f7333addf107c773b78f74359ef1fec262d5a22eeb9818db1512e1d0aab30f1baa00944bdd49eb7d8717f02454064b
+  metadata.gz: 859196aa39020d924fa7af4df6d96c110f41ac2b90a39dc89ed6935fc64e857b2bffb5776a366660ab61c55a96dd35b9bd6663ec23c7ee4249cae3103bc0a2aa
+  data.tar.gz: b07ec53917af58e737058c504685d283850e072f0794c457bd961d39b9815c85b2fc2a9bed4de2a643675dc0e0f7bb2077b4c41b2c28c9c94f948a532baae6bb

data/README.md

@@ -18,7 +18,7 @@ You can use datasets easily because you can access each dataset with multiple wa
 
 ## Usage
 
-Here is an example to access iris dataset by `#each`:
+Here is an example to access [Iris Data Set](https://archive.ics.uci.edu/ml/datasets/iris) by `#each`  or `Table#to_h` or `Table#fetch_values`.
 
 ```ruby
 require "datasets"
@@ -32,12 +32,40 @@ iris.each do |record|
      record.petal_width,
      record.class,
   ]
-  # [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]
-  # [7.0, 3.2, 4.7, 1.4, "Iris-versicolor"]
 end
+# => [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]
+# => [4.9, 3.0, 1.4, 0.2, "Iris-setosa"]
+  :
+# => [7.0, 3.2, 4.7, 1.4, "Iris-versicolor"]
+
+
+iris_hash = iris.to_table.to_h
+p iris_hash[:sepal_length]
+# => [5.1, 4.9, .. , 7.0, ..
+p iris_hash[:sepal_width]
+# => [3.5, 3.0, .. , 3.2, ..
+p iris_hash[:petal_length]
+# => [1.4, 1.4, .. , 4.7, ..
+p iris_hash[:petal_width]
+# => [0.2, 0.2, .. , 1.4, ..
+p iris_hash[:class]
+# => ["Iris-setosa", "Iris-setosa", .. , "Iris-versicolor", ..
+
+
+iris_table = iris.to_table
+p iris_table.fetch_values(:sepal_length, :sepal_width, :petal_length, :petal_width).transpose
+# => [[5.1, 3.5, 1.4, 0.2],
+      [4.9, 3.0, 1.4, 0.2],
+      :
+      [7.0, 3.2, 4.7, 1.4],
+      :
+
+p iris_table[:class]
+# => ["Iris-setosa", "Iris-setosa", .. , "Iris-versicolor", ..
 ```
 
-Here is an example to access CIFAR dataset by `#each`:
+
+Here is an example to access [The CIFAR-10/100 dataset](https://www.cs.toronto.edu/~kriz/cifar.html) by `#each`:
 
 **CIFAR-10**
 
@@ -73,6 +101,24 @@ cifar.each do |record|
 end
 ```
 
+**MNIST**
+
+```ruby
+require "datasets"
+
+mnist = Datasets::MNIST.new(type: :train)
+mnist.metadata
+#=> #<struct Datasets::Metadata name="MNIST-train", url="http://yann.lecun.com/exdb/mnist/", licenses=nil, description="a training set of 60,000 examples">
+
+mnist.each do |record|
+  p record.pixels
+  # => [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, .....]
+  p record.label
+  # => 5
+end
+```
+
+
 ## License
 
 The MIT license. See `LICENSE.txt` for details.

data/doc/text/news.md

@@ -1,5 +1,13 @@
 # News
 
+## 0.0.6 - 2018-07-25
+
+### Improvements
+
+  * `Datasets::MNIST`: Added.
+
+  * `Datasets::PennTreebank`: Added.
+
 ## 0.0.5 - 2018-06-06
 
 ### Improvements

data/lib/datasets/cifar.rb

@@ -27,14 +27,17 @@ module Datasets
 
     def initialize(n_classes: 10, type: :train)
       unless [10, 100].include?(n_classes)
-        raise 'Please set n_classes 10 or 100'
+        message = "Please set n_classes 10 or 100: #{n_classes.inspect}"
+        raise ArgumentError, message
       end
       unless [:train, :test].include?(type)
-        raise 'Please set type :train or :test'
+        message = "Please set type :train or :test: #{type.inspect}"
+        raise ArgumentError, message
       end
 
       super()
 
+      @metadata.id = "cifar-#{n_classes}"
       @metadata.name = "CIFAR-#{n_classes}"
       @metadata.url = "https://www.cs.toronto.edu/~kriz/cifar.html"
       @metadata.description = "CIFAR-#{n_classes} is 32x32 image dataset"

data/lib/datasets/dataset.rb

@@ -27,7 +27,7 @@ module Datasets
       else
         base_dir = ENV["XDG_CACHE_HOME"] || "~/.cache"
       end
-      Pathname(base_dir).expand_path + "red-datasets" + metadata.name
+      Pathname(base_dir).expand_path + "red-datasets" + metadata.id
     end
 
     def download(output_path, url)

data/lib/datasets/iris.rb

@@ -12,7 +12,8 @@ module Datasets
 
     def initialize
       super()
-      @metadata.name = "iris"
+      @metadata.id = "iris"
+      @metadata.name = "Iris"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/Iris"
       @metadata.description = lambda do
         read_names

data/lib/datasets/metadata.rb

@@ -1,5 +1,6 @@
 module Datasets
-  class Metadata < Struct.new(:name,
+  class Metadata < Struct.new(:id,
+                              :name,
                               :url,
                               :licenses,
                               :description)

data/lib/datasets/mnist.rb

@@ -0,0 +1,108 @@
+require 'zlib'
+
+require_relative "dataset"
+
+class SetTypeError < StandardError; end
+
+module Datasets
+  class MNIST < Dataset
+
+    class Record < Struct.new(:data, :label)
+      def pixels
+        data.unpack("C*")
+      end
+
+      def to_h
+        hash = super
+        hash[:pixels] = pixels
+        hash
+      end
+    end
+
+    def initialize(type: :train)
+      unless [:train, :test].include?(type)
+        raise ArgumentError, "Please set type :train or :test: #{type.inspect}"
+      end
+
+      super()
+
+      @metadata.id = "mnist-#{type}"
+      @metadata.name = "MNIST: #{type}"
+      @metadata.url = "http://yann.lecun.com/exdb/mnist/"
+      @type = type
+
+      case type
+      when :train
+        @metadata.description = "a training set of 60,000 examples"
+      when :test
+        @metadata.description = "a test set of 10,000 examples"
+      end
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      image_path = cache_dir_path + target_file(:image)
+      label_path = cache_dir_path + target_file(:label)
+      base_url = "http://yann.lecun.com/exdb/mnist/"
+
+      unless image_path.exist?
+        download(image_path, base_url + target_file(:image))
+      end
+
+      unless label_path.exist?
+        download(label_path, base_url + target_file(:label))
+      end
+
+      open_data(image_path, label_path, &block)
+    end
+
+    private
+    def open_data(image_path, label_path, &block)
+      labels = parse_labels(label_path)
+
+      Zlib::GzipReader.open(image_path) do |f|
+        n_uint32s = 4
+        n_bytes = n_uint32s * 4
+        mnist_magic_number = 2051
+        magic, n_images, n_rows, n_cols = f.read(n_bytes).unpack("N*")
+        raise 'This is not MNIST image file' if magic != mnist_magic_number
+        n_images.times do |i|
+          data = f.read(n_rows * n_cols)
+          label = labels[i]
+          yield Record.new(data, label)
+        end
+      end
+    end
+
+    def target_file(data)
+      case @type
+      when :train
+        case data
+        when :image
+          "train-images-idx3-ubyte.gz"
+        when :label
+          "train-labels-idx1-ubyte.gz"
+        end
+      when :test
+        case data
+        when :image
+          "t10k-images-idx3-ubyte.gz"
+        when :label
+          "t10k-labels-idx1-ubyte.gz"
+        end
+      end
+    end
+
+    def parse_labels(file_path)
+      Zlib::GzipReader.open(file_path) do |f|
+        n_uint32s = 4
+        n_bytes = n_uint32s * 2
+        mnist_magic_number = 2049
+        magic, n_labels = f.read(n_bytes).unpack('N2')
+        raise 'This is not MNIST label file' if magic != mnist_magic_number
+        f.read(n_labels).unpack('C*')
+      end
+    end
+  end
+end

data/lib/datasets/penn-treebank.rb

@@ -0,0 +1,65 @@
+require_relative "dataset"
+
+module Datasets
+  class PennTreebank < Dataset
+    Record = Struct.new(:word, :id)
+
+    DESCRIPTION = <<~DESC
+      `Penn Tree Bank <https://www.cis.upenn.edu/~treebank/>`_ is originally a
+      corpus of English sentences with linguistic structure annotations. This
+      function uses a variant distributed at
+      `https://github.com/wojzaremba/lstm <https://github.com/wojzaremba/lstm>`_,
+      which omits the annotation and splits the dataset into three parts:
+      training, validation, and test.
+    DESC
+
+    def initialize(type: :train)
+      valid_types = [:train, :test, :valid]
+      unless valid_types.include?(type)
+        valid_types_label = valid_types.collect(&:inspect).join(", ")
+        message = "Type must be one of [#{valid_types_label}]: #{type.inspect}"
+        raise ArgumentError, message
+      end
+      @type = type
+
+      super()
+
+      @metadata.id = "penn-treebank-#{@type}"
+      @metadata.name = "Penn Treebank: #{@type}"
+      @metadata.description = DESCRIPTION
+      @metadata.url = "https://github.com/wojzaremba/lstm"
+      @metadata.licenses = ["Apache-2.0"]
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      base_name = "ptb.#{@type}.txt"
+      data_path = cache_dir_path + base_name
+      unless data_path.exist?
+        base_url = "https://raw.githubusercontent.com/wojzaremba/lstm/master/data"
+        download(data_path, "#{base_url}/#{base_name}")
+      end
+
+      parse_data(data_path, &block)
+    end
+
+    private
+    def parse_data(data_path)
+      index = 0
+      vocabulary = {}
+      File.open(data_path) do |f|
+        f.each_line do |line|
+          line.split.each do |word|
+            word = word.strip
+            unless vocabulary.key?(word)
+              vocabulary[word] = index
+              index += 1
+            end
+            yield(Record.new(word, vocabulary[word]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/datasets/wikipedia.rb

@@ -30,14 +30,15 @@ module Datasets
       super()
       @language = language
       @type = type
-      @metadata.name = "wikipedia-#{@language}-#{@type}"
+      @metadata.id = "wikipedia-#{@language}-#{@type}"
+      @metadata.name = "Wikipedia #{@type} (#{@language})"
       @metadata.url = "https://dumps.wikimedia.org/"
       @metadata.licenses = [
         "CC-BY-SA-3.0",
         "CC-BY-SA-4.0",
         "GFDL-1.3-or-later",
       ]
-      @metadata.description = "Wikipedia #{@type} (#{@language})"
+      @metadata.description = "Wikipedia #{@type} in #{@language}"
     end
 
     def each(&block)

data/lib/datasets.rb

@@ -2,4 +2,6 @@ require "datasets/version"
 
 require "datasets/cifar"
 require "datasets/iris"
+require "datasets/mnist"
+require "datasets/penn-treebank"
 require "datasets/wikipedia"

data/test/helper.rb

@@ -13,6 +13,7 @@ module Helper
     end
 
     def teardown_sandbox
+      return unless defined?(@tmp_dir)
       FileUtils.rm_rf(@tmp_dir)
     end
   end

data/test/test-cifar.rb

@@ -216,4 +216,14 @@ class CIFARTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("invalid") do
+    test("type") do
+      invalid_type = :invalid
+      message = "Please set type :train or :test: #{invalid_type.inspect}"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::CIFAR.new(type: invalid_type)
+      end
+    end
+  end
 end

data/test/test-mnist.rb

@@ -0,0 +1,111 @@
+class MNISTTest < Test::Unit::TestCase
+  include Helper::Sandbox
+
+  sub_test_case("Normal") do
+    def setup_data
+      setup_sandbox
+
+      def @dataset.cache_dir_path
+        @cache_dir_path
+      end
+
+      def @dataset.cache_dir_path=(path)
+        @cache_dir_path = path
+      end
+      @dataset.cache_dir_path = @tmp_dir
+
+      def @dataset.download(output_path, url)
+        image_magic_number = 2051
+        label_magic_number = 2049
+        n_image, image_size_x, image_size_y, label = 10, 28, 28, 1
+
+        Zlib::GzipWriter.open(output_path) do |gz|
+          if output_path.basename.to_s.include?("-images-")
+            image_data = ([image_magic_number, n_image]).pack('N2') +
+                         ([image_size_x,image_size_y]).pack('N2') +
+                         ([0] * image_size_x * image_size_y).pack("C*") * n_image
+            gz.puts(image_data)
+          else
+            label_data = ([label_magic_number, n_image]).pack('N2') +
+                         ([label] * n_image).pack("C*")
+            gz.puts(label_data)
+          end
+        end
+      end
+    end
+
+    def teardown
+      teardown_sandbox
+    end
+
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::MNIST.new(type: :train)
+        setup_data()
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :label => record.label,
+            :pixels => record.pixels
+          }
+        end
+
+        assert_equal([
+                       {
+                         :label => 1,
+                         :pixels => [0] * 28 * 28
+                       }
+                     ] * 10,
+                     raw_dataset)
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([[0] * 28 * 28] * 10,
+                     table_data[:pixels])
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::MNIST.new(type: :test)
+        setup_data()
+      end
+
+      test("#each") do
+        raw_dataset = @dataset.collect do |record|
+          {
+            :label => record.label,
+            :pixels => record.pixels
+          }
+        end
+
+        assert_equal([
+                       {
+                         :label => 1,
+                         :pixels => [0] * 28 * 28
+                       }
+                     ] * 10,
+                     raw_dataset)
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([[0] * 28 * 28] * 10,
+                     table_data[:pixels])
+      end
+    end
+  end
+
+  sub_test_case("Abnormal") do
+    test("invalid type") do
+      invalid_type = :invalid
+      message = "Please set type :train or :test: #{invalid_type.inspect}"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::MNIST.new(type: invalid_type)
+      end
+    end
+  end
+end

data/test/test-penn-treebank.rb

@@ -0,0 +1,59 @@
+class PennTreebankTest < Test::Unit::TestCase
+  def record(*args)
+    Datasets::PennTreebank::Record.new(*args)
+  end
+
+  sub_test_case("type") do
+    test("train") do
+      dataset = Datasets::PennTreebank.new(type: :train)
+      records = dataset.to_a
+      assert_equal([
+                     887521,
+                     record("aer", 0),
+                     record("<unk>", 25),
+                   ],
+                   [
+                     records.size,
+                     records[0],
+                     records[-1],
+                   ])
+    end
+
+    test("test") do
+      dataset = Datasets::PennTreebank.new(type: :test)
+      records = dataset.to_a
+      assert_equal([
+                     78669,
+                     record("no", 0),
+                     record("us", 953),
+                   ],
+                   [
+                     records.size,
+                     records[0],
+                     records[-1],
+                   ])
+    end
+
+    test("valid") do
+      dataset = Datasets::PennTreebank.new(type: :valid)
+      records = dataset.to_a
+      assert_equal([
+                     70390,
+                     record("consumers", 0),
+                     record("N", 28),
+                   ],
+                   [
+                     records.size,
+                     records[0],
+                     records[-1],
+                   ])
+    end
+
+    test("invalid") do
+      message = "Type must be one of [:train, :test, :valid]: :invalid"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::PennTreebank.new(type: :invalid)
+      end
+    end
+  end
+end

data/test/test-wikipedia.rb

@@ -83,13 +83,18 @@ class WikipediaTest < Test::Unit::TestCase
       end
 
       sub_test_case("#metadata") do
-        test("#name") do
+        test("#id") do
           assert_equal("wikipedia-ja-articles",
+                       @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Wikipedia articles (ja)",
                        @dataset.metadata.name)
         end
 
         test("#description") do
-          assert_equal("Wikipedia articles (ja)",
+          assert_equal("Wikipedia articles in ja",
                        @dataset.metadata.description)
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - tomisuker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-06 00:00:00.000000000 Z
+date: 2018-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -104,6 +104,8 @@ files:
 - lib/datasets/downloader.rb
 - lib/datasets/iris.rb
 - lib/datasets/metadata.rb
+- lib/datasets/mnist.rb
+- lib/datasets/penn-treebank.rb
 - lib/datasets/table.rb
 - lib/datasets/version.rb
 - lib/datasets/wikipedia.rb
@@ -112,6 +114,8 @@ files:
 - test/run-test.rb
 - test/test-cifar.rb
 - test/test-iris.rb
+- test/test-mnist.rb
+- test/test-penn-treebank.rb
 - test/test-table.rb
 - test/test-wikipedia.rb
 homepage: https://github.com/red-data-tools/red-datasets
@@ -139,9 +143,11 @@ signing_key:
 specification_version: 4
 summary: Red Datasets provides classes that provide common datasets such as iris dataset.
 test_files:
-- test/test-iris.rb
+- test/run-test.rb
+- test/test-cifar.rb
 - test/test-wikipedia.rb
+- test/test-iris.rb
 - test/helper.rb
-- test/run-test.rb
+- test/test-mnist.rb
 - test/test-table.rb
-- test/test-cifar.rb
+- test/test-penn-treebank.rb