red-datasets 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0239c4ab86dd9f589b1f67b9d6c381570e25a29289c261470943ed48f7dfc3d0'
-  data.tar.gz: 2f3f3af1f17a1bd1e7aa307e2b182108790549754d907262105e18479997cde6
+  metadata.gz: 6fbd4d11063f89ba2e09250b751886086c953ec8bc92c75a6a351c31a36da0c4
+  data.tar.gz: acc6ff31f0f4ae3a6c6565fe569233c01615718c01300b0838ff744571edc34d
 SHA512:
-  metadata.gz: 04b3dbc23dc8679855a6104a9f3da39871594979f149295ef13b3be864a3dbbdb6bec3fb59153db9b5be4fade6819686e13b60a38f1d1721bf7e1163d4bb49b8
-  data.tar.gz: 476a9081fe0db32aad8a4e00c7e08f77002e58a2f2c68eb37aecf2a70054d43877707ddda29137b361a5a37ff979f3c28c5f2a03c2d1e96cbc7f7289f659ba9f
+  metadata.gz: 26361511155b447ffed56a79b2336a9a1db96494bf856b23e7b39cc6a8b6a2039e7ed27564140761bdb2daaae7ee563b3695c464a7a7b21ff93b0636f6b8338d
+  data.tar.gz: 40446f90e410e0d86abeec186a1d7adcc5375e29c19dc934f823befb26a87d904458ef5ea18c9d64055493d29ed305dba53d6e4d86bd7d84488baf3745ebd792

data/doc/text/news.md

@@ -1,5 +1,14 @@
 # News
 
+## 0.1.2 - 2021-06-03
+
+### Improvements
+
+  * `Datasets::Rdatasets` and `Datasets::RdatasetsList`: Added.
+
+  * `Datasets::Penguins`: Changed for compatibility with seaborn's
+    penguins dataset.
+
 ## 0.1.1 - 2021-04-11
 
 ### Improvements

data/lib/datasets.rb

@@ -15,5 +15,6 @@ require_relative "datasets/mushroom"
 require_relative "datasets/penguins"
 require_relative "datasets/penn-treebank"
 require_relative "datasets/postal-code-japan"
+require_relative "datasets/rdatasets"
 require_relative "datasets/wikipedia"
 require_relative "datasets/wine"

data/lib/datasets/cifar.rb

@@ -1,10 +1,10 @@
-require "rubygems/package"
-require "zlib"
-
+require_relative "tar_gz_readable"
 require_relative "dataset"
 
 module Datasets
   class CIFAR < Dataset
+    include TarGzReadable
+
     module Pixelable
       def pixels
         data.unpack("C*")
@@ -61,7 +61,7 @@ module Datasets
     private
 
     def parse_data(data_path, &block)
-      open_tar(data_path) do |tar|
+      open_tar_gz(data_path) do |tar|
         target_file_names.each do |target_file_name|
           tar.seek(target_file_name) do |entry|
             parse_entry(entry, &block)
@@ -124,14 +124,6 @@ module Datasets
         end
       end
     end
-
-    def open_tar(data_path)
-      Zlib::GzipReader.open(data_path) do |f|
-        Gem::Package::TarReader.new(f) do |tar|
-          yield(tar)
-        end
-      end
-    end
   end
 end
 

data/lib/datasets/dataset.rb

@@ -18,11 +18,17 @@ module Datasets
       Table.new(self)
     end
 
+    def clear_cache!
+      if cache_dir_path.exist?
+        FileUtils.rmtree(cache_dir_path.to_s, secure: true)
+      end
+    end
+
     private
     def cache_dir_path
       case RUBY_PLATFORM
       when /mswin/, /mingw/
-        base_dir = ENV["LOCALAPPDATA"] || "~/AppData"
+        base_dir = ENV["LOCALAPPDATA"] || "~/AppData/Local"
       when /darwin/
         base_dir = "~/Library/Caches"
       else

data/lib/datasets/downloader.rb

@@ -8,6 +8,8 @@ require "pathname"
 
 module Datasets
   class Downloader
+    class TooManyRedirects < StandardError; end
+
     def initialize(url)
       if url.is_a?(URI::Generic)
         url = url.dup
@@ -31,41 +33,65 @@ module Datasets
         headers["Range"] = "bytes=#{start}-"
       end
 
-      Net::HTTP.start(@url.hostname,
-                      @url.port,
-                      :use_ssl => (@url.scheme == "https")) do |http|
-        path = @url.path
-        path += "?#{@url.query}" if @url.query
+      start_http(@url, headers) do |response|
+        if response.is_a?(Net::HTTPPartialContent)
+          mode = "ab"
+        else
+          start = nil
+          mode = "wb"
+        end
+
+        base_name = @url.path.split("/").last
+        size_current = 0
+        size_max = response.content_length
+        if start
+          size_current += start
+          size_max += start
+        end
+        progress_reporter = ProgressReporter.new(base_name, size_max)
+        partial_output_path.open(mode) do |output|
+          response.read_body do |chunk|
+            size_current += chunk.bytesize
+            progress_reporter.report(size_current)
+            output.write(chunk)
+          end
+        end
+      end
+      FileUtils.mv(partial_output_path, output_path)
+    rescue TooManyRedirects => error
+      last_url = error.message[/\Atoo many redirections: (.+)\z/, 1]
+      raise TooManyRedirects, "too many redirections: #{@url} .. #{last_url}"
+    end
+
+    private def start_http(url, headers, limit = 10, &block)
+      if limit == 0
+        raise TooManyRedirects, "too many redirections: #{url}"
+      end
+      http = Net::HTTP.new(url.hostname, url.port)
+      # http.set_debug_output($stderr)
+      http.use_ssl = (url.scheme == "https")
+      http.start do
+        path = url.path
+        path += "?#{url.query}" if url.query
         request = Net::HTTP::Get.new(path, headers)
         http.request(request) do |response|
           case response
-          when Net::HTTPPartialContent
-            mode = "ab"
-          when Net::HTTPSuccess
-            start = nil
-            mode = "wb"
+          when Net::HTTPSuccess, Net::HTTPPartialContent
+            return block.call(response)
+          when Net::HTTPRedirection
+            url = URI.parse(response[:location])
+            $stderr.puts "Redirect to #{url}"
+            return start_http(url, headers, limit - 1, &block)
           else
-            break
-          end
-
-          base_name = @url.path.split("/").last
-          size_current = 0
-          size_max = response.content_length
-          if start
-            size_current += start
-            size_max += start
-          end
-          progress_reporter = ProgressReporter.new(base_name, size_max)
-          partial_output_path.open(mode) do |output|
-            response.read_body do |chunk|
-              size_current += chunk.bytesize
-              progress_reporter.report(size_current)
-              output.write(chunk)
+            message = response.code
+            if response.message and not response.message.empty?
+              message += ": #{response.message}"
             end
+            message += ": #{url}"
+            raise response.error_type.new(message, response)
           end
         end
       end
-      FileUtils.mv(partial_output_path, output_path)
     end
 
     class ProgressReporter

data/lib/datasets/penguins.rb

@@ -19,7 +19,6 @@ module Datasets
                         :delta_15_n_permil,
                         :delta_13_c_permil,
                         :comments)
-
     class SpeciesBase < Dataset
       def initialize
         super
@@ -62,17 +61,17 @@ module Datasets
       URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
     end
 
-    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
-    class Gentoo < SpeciesBase
-      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
-    end
-
     # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
     class Chinstrap < SpeciesBase
       DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
       URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
     end
+
+    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
+    class Gentoo < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+    end
   end
 
   # This dataset provides the same dataset as https://github.com/allisonhorst/palmerpenguins
@@ -100,8 +99,8 @@ module Datasets
 
       species_classes = [
         PenguinsRawData::Adelie,
+        PenguinsRawData::Chinstrap,
         PenguinsRawData::Gentoo,
-        PenguinsRawData::Chinstrap
       ]
 
       species_classes.each do |species_class|
@@ -112,14 +111,36 @@ module Datasets
     end
 
     private def convert_record(raw_record)
-      Record.new(raw_record.species.split(' ')[0],
-                 raw_record.island,
-                 raw_record.culmen_length_mm,
-                 raw_record.culmen_depth_mm,
-                 raw_record.flipper_length_mm&.to_i,
-                 raw_record.body_mass_g&.to_i,
-                 raw_record.sex&.downcase,
-                 raw_record.date_egg&.year)
+      Record.new(*cleanse_fields(raw_record))
+    end
+
+    private def cleanse_fields(raw_record)
+      species = raw_record.species.split(' ')[0]
+      flipper_length_mm = raw_record.flipper_length_mm&.to_i
+      body_mass_g = raw_record.body_mass_g&.to_i
+      sex = normalize_sex(raw_record.sex)
+      year = raw_record.date_egg&.year
+
+      [
+        species,
+        raw_record.island,
+        raw_record.culmen_length_mm,
+        raw_record.culmen_depth_mm,
+        flipper_length_mm,
+        body_mass_g,
+        sex,
+        year
+      ]
+    end
+
+    private def normalize_sex(val)
+      val = val&.downcase
+      case val
+      when "female", "male", nil
+        val
+      else
+        nil
+      end
     end
   end
 end

data/lib/datasets/rdatasets.rb

@@ -0,0 +1,95 @@
+require_relative "dataset"
+require_relative "tar_gz_readable"
+
+module Datasets
+  class RdatasetsList < Dataset
+    Record = Struct.new(:package,
+                        :dataset,
+                        :title,
+                        :rows,
+                        :cols,
+                        :n_binary,
+                        :n_character,
+                        :n_factor,
+                        :n_logical,
+                        :n_numeric,
+                        :csv,
+                        :doc)
+
+    def initialize
+      super
+      @metadata.id = "rdatasets"
+      @metadata.name = "Rdatasets"
+      @metadata.url = "https://vincentarelbundock.github.io/Rdatasets/"
+      @metadata.licenses = ["GPL-3"]
+      @data_url = "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/datasets.csv"
+      @data_path = cache_dir_path + "datasets.csv"
+    end
+
+    def filter(package: nil, dataset: nil)
+      return to_enum(__method__, package: package, dataset: dataset) unless block_given?
+
+      conds = {}
+      conds["Package"] = package if package
+      conds["Item"]    = dataset if dataset
+      if conds.empty?
+        each_row {|row| yield Record.new(*row.fields) }
+      else
+        each_row do |row|
+          if conds.all? {|k, v| row[k] == v }
+            yield Record.new(*row.fields)
+          end
+        end
+      end
+    end
+
+    def each(&block)
+      filter(&block)
+    end
+
+    private def each_row(&block)
+      download(@data_path, @data_url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each(&block)
+      end
+    end
+  end
+
+  class Rdatasets < Dataset
+    def initialize(package_name, dataset_name)
+      list = RdatasetsList.new
+
+      info = list.filter(package: package_name, dataset: dataset_name).first
+      unless info
+        raise ArgumentError, "Unable to locate dataset #{package_name}/#{dataset_name}"
+      end
+
+      super()
+      @metadata.id = "rdatasets-#{package_name}-#{dataset_name}"
+      @metadata.name = "Rdatasets: #{package_name}: #{dataset_name}"
+      @metadata.url = info.csv
+      @metadata.licenses = ["GPL-3"]
+      @metadata.description = info.title
+
+      # Follow the original directory structure in the cache directory
+      @data_path = cache_dir_path + (dataset_name + ".csv")
+
+      @package_name = package_name
+      @dataset_name = dataset_name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      download(@data_path, @metadata.url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = row.to_h
+          record.delete("")
+          record.transform_keys!(&:to_sym)
+          yield record
+        end
+      end
+    end
+  end
+end

data/lib/datasets/tar_gz_readable.rb

@@ -0,0 +1,14 @@
+require "rubygems/package"
+require "zlib"
+
+module Datasets
+  module TarGzReadable
+    def open_tar_gz(data_path)
+      Zlib::GzipReader.open(data_path) do |f|
+        Gem::Package::TarReader.new(f) do |tar|
+          yield(tar)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/test/test-dataset.rb

@@ -0,0 +1,27 @@
+class TestDataset < Test::Unit::TestCase
+  sub_test_case("#clear_cache!") do
+    def setup
+      @dataset = Datasets::Iris.new
+      @cache_dir_path = @dataset.send(:cache_dir_path)
+    end
+
+    test("when the dataset is downloaded") do
+      @dataset.first # This ensures the dataset downloaded
+      existence = {before: @cache_dir_path.join("iris.csv").exist?}
+
+      @dataset.clear_cache!
+      existence[:after] = @cache_dir_path.join("iris.csv").exist?
+
+      assert_equal({before: true, after: false},
+                   existence)
+    end
+
+    test("when the dataset is not downloaded") do
+      FileUtils.rmtree(@cache_dir_path.to_s, secure: true) if @cache_dir_path.exist?
+
+      assert_nothing_raised do
+        @dataset.clear_cache!
+      end
+    end
+  end
+end

data/test/test-downloader.rb

@@ -0,0 +1,29 @@
+class DownloaderTest < Test::Unit::TestCase
+  include Helper::Sandbox
+
+  sub_test_case("#download") do
+    def setup
+      setup_sandbox
+    end
+
+    def teardown
+      teardown_sandbox
+    end
+
+    test("too many redirection") do
+      first_url = "https://example.com/file"
+      last_url = "https://example.com/last_redirection"
+      expected_message = "too many redirections: #{first_url} .. #{last_url}"
+      output_path = @tmp_dir + "file"
+      downloader = Datasets::Downloader.new(first_url)
+
+      downloader.define_singleton_method(:start_http) do |url, headers|
+        raise Datasets::Downloader::TooManyRedirects, "too many redirections: #{last_url}"
+      end
+
+      assert_raise(Datasets::Downloader::TooManyRedirects.new(expected_message)) do
+        downloader.download(output_path)
+      end
+    end
+  end
+end

data/test/test-penguins.rb

@@ -182,6 +182,18 @@ class PenguinsTest < Test::Unit::TestCase
       @dataset = Datasets::Penguins.new
     end
 
+    test("order of species") do
+      species_values = @dataset.map {|r| r.species }.uniq
+      assert_equal(["Adelie", "Chinstrap", "Gentoo"],
+                   species_values)
+    end
+
+    test("data cleansing") do
+      sex_values = @dataset.map {|r| r.sex }.uniq.compact.sort
+      assert_equal(["female", "male"],
+                   sex_values)
+    end
+
     test("#each") do
       records = @dataset.each.to_a
       assert_equal([
@@ -196,16 +208,6 @@ class PenguinsTest < Test::Unit::TestCase
                        sex: "male",
                        year: 2007
                      },
-                     {
-                       species: "Gentoo",
-                       island: "Biscoe",
-                       bill_length_mm: 46.1,
-                       bill_depth_mm: 13.2,
-                       flipper_length_mm: 211,
-                       body_mass_g: 4500,
-                       sex: "female",
-                       year: 2007
-                     },
                      {
                        species: "Chinstrap",
                        island: "Dream",
@@ -217,13 +219,23 @@ class PenguinsTest < Test::Unit::TestCase
                        year: 2007
                      },
                      {
-                       species: "Chinstrap",
-                       island: "Dream",
-                       bill_length_mm: 50.2,
-                       bill_depth_mm: 18.7,
-                       flipper_length_mm: 198,
-                       body_mass_g: 3775,
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 46.1,
+                       bill_depth_mm: 13.2,
+                       flipper_length_mm: 211,
+                       body_mass_g: 4500,
                        sex: "female",
+                       year: 2007
+                     },
+                     {
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 49.9,
+                       bill_depth_mm: 16.1,
+                       flipper_length_mm: 213,
+                       body_mass_g: 5400,
+                       sex: "male",
                        year: 2009
                      }
                    ],
@@ -231,7 +243,7 @@ class PenguinsTest < Test::Unit::TestCase
                      records.size,
                      records[0].to_h,
                      records[152].to_h,
-                     records[276].to_h,
+                     records[220].to_h,
                      records[-1].to_h,
                    ])
     end

data/test/test-rdatasets.rb

@@ -0,0 +1,136 @@
+class RdatasetsTest < Test::Unit::TestCase
+  sub_test_case("RdatasetsList") do
+    def setup
+      @dataset = Datasets::RdatasetsList.new
+    end
+
+    sub_test_case("#each") do
+      test("with package_name") do
+        records = @dataset.filter(package: "datasets").to_a
+        assert_equal([
+                       84,
+                       {
+                         package: "datasets",
+                         dataset: "ability.cov",
+                         title: "Ability and Intelligence Tests",
+                         rows: 6,
+                         cols: 8,
+                         n_binary: 0,
+                         n_character: 0,
+                         n_factor: 0,
+                         n_logical: 0,
+                         n_numeric: 8,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/ability.cov.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/ability.cov.html"
+                       },
+                       {
+                         package: "datasets",
+                         dataset: "WWWusage",
+                         title: "Internet Usage per Minute",
+                         rows: 100,
+                         cols: 2,
+                         n_binary: 0,
+                         n_character: 0,
+                         n_factor: 0,
+                         n_logical: 0,
+                         n_numeric: 2,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/WWWusage.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/WWWusage.html"
+                       }
+                     ],
+                     [
+                       records.size,
+                       records[0].to_h,
+                       records[-1].to_h
+                     ])
+      end
+
+      test("without package_name") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       1478,
+                       {
+                         package: "AER",
+                         dataset: "Affairs",
+                         title: "Fair's Extramarital Affairs Data",
+                         rows: 601,
+                         cols: 9,
+                         n_binary: 2,
+                         n_character: 0,
+                         n_factor: 2,
+                         n_logical: 0,
+                         n_numeric: 7,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/AER/Affairs.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/AER/Affairs.html"
+                       },
+                       {
+                         package: "vcd",
+                         dataset: "WomenQueue",
+                         title: "Women in Queues",
+                         rows: 11,
+                         cols: 2,
+                         n_binary: 0,
+                         n_character: 0,
+                         n_factor: 1,
+                         n_logical: 0,
+                         n_numeric: 1,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/vcd/WomenQueue.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/vcd/WomenQueue.html"
+                       },
+                     ],
+                     [
+                       records.size,
+                       records[0].to_h,
+                       records[-1].to_h
+                     ])
+      end
+    end
+  end
+
+  sub_test_case("Rdatasets") do
+    sub_test_case("datasets") do
+      sub_test_case("AirPassengers") do
+        def setup
+          @dataset = Datasets::Rdatasets.new("datasets", "AirPassengers")
+        end
+
+        test("#each") do
+          records = @dataset.each.to_a
+          assert_equal([
+                         144,
+                         { time: 1949,             value: 112 },
+                         { time: 1960.91666666667, value: 432 },
+                       ],
+                       [
+                         records.size,
+                         records[0],
+                         records[-1]
+                       ])
+        end
+
+        test("#metadata.id") do
+          assert_equal("rdatasets-datasets-AirPassengers", @dataset.metadata.id)
+        end
+
+        test("#metadata.description") do
+          description = @dataset.metadata.description
+          assert do
+            description.include?("Monthly Airline Passenger Numbers 1949-1960")
+          end
+        end
+      end
+
+      test("invalid dataset name") do
+        assert_raise(ArgumentError) do
+          Datasets::Rdatasets.new("datasets", "invalid datasets name")
+        end
+      end
+    end
+
+    test("invalid package name") do
+      assert_raise(ArgumentError) do
+        Datasets::Rdatasets.new("invalid package name", "AirPassengers")
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - tomisuker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-11 00:00:00.000000000 Z
+date: 2021-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -161,7 +161,9 @@ files:
 - lib/datasets/penguins.rb
 - lib/datasets/penn-treebank.rb
 - lib/datasets/postal-code-japan.rb
+- lib/datasets/rdatasets.rb
 - lib/datasets/table.rb
+- lib/datasets/tar_gz_readable.rb
 - lib/datasets/version.rb
 - lib/datasets/wikipedia.rb
 - lib/datasets/wine.rb
@@ -172,7 +174,9 @@ files:
 - test/test-cifar.rb
 - test/test-cldr-plurals.rb
 - test/test-communities.rb
+- test/test-dataset.rb
 - test/test-dictionary.rb
+- test/test-downloader.rb
 - test/test-e-stat-japan.rb
 - test/test-fashion-mnist.rb
 - test/test-hepatitis.rb
@@ -184,6 +188,7 @@ files:
 - test/test-penguins.rb
 - test/test-penn-treebank.rb
 - test/test-postal-code-japan.rb
+- test/test-rdatasets.rb
 - test/test-table.rb
 - test/test-wikipedia.rb
 - test/test-wine.rb
@@ -217,7 +222,9 @@ test_files:
 - test/test-cifar.rb
 - test/test-cldr-plurals.rb
 - test/test-communities.rb
+- test/test-dataset.rb
 - test/test-dictionary.rb
+- test/test-downloader.rb
 - test/test-e-stat-japan.rb
 - test/test-fashion-mnist.rb
 - test/test-hepatitis.rb
@@ -229,6 +236,7 @@ test_files:
 - test/test-penguins.rb
 - test/test-penn-treebank.rb
 - test/test-postal-code-japan.rb
+- test/test-rdatasets.rb
 - test/test-table.rb
 - test/test-wikipedia.rb
 - test/test-wine.rb