red-datasets 0.0.9 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81ed53e83d75d517052aaf07c66fe177f12f986584141c951ac1dcfa2fc88646
-  data.tar.gz: 94b9f3b8042eaad65304bf7c3d2fc35519f8328b0ca4e9f8a7ad9be13781a91e
+  metadata.gz: 120492172aae9cec1c4fc4f3b73575cb5349caf2f0b67d70676c8896324e1491
+  data.tar.gz: e46eb3f2875cb407e86cc0976eff7d612beb62ca6b421a51435b5d5e1bfa6e03
 SHA512:
-  metadata.gz: c73561ed005e4b58f27fc6de969605a22d57adf4bc5b5184e5cdb65739f1ac6b86f6ed67794bfe61164859fc4a1b0f80430bc819b2ea37ac455a560a6f008b13
-  data.tar.gz: 07560b09d68272dc7a959c16ec03975d1fa752f9d6930f0fd746c46e9236995606694f5899bad5bf770812c5a2d81e6f013353f680fc8adf65ad42bae514f57c
+  metadata.gz: 360bbf78c131f20a67359ddc2055cd58502da1f4e95adf30475cd405d5eb50be6ba4fd9aa0a0857226dc803e14282cc4231de113843e96657a65e287c7500137
+  data.tar.gz: f88ed1ae8c8f0dad9f4d8904a265c833ceee723ba92860c0e3bed4c193d56a901c31184abd4290058de47fbc089b12b4d3b1da064f138214e2954d45eee928da

data/doc/text/news.md

@@ -1,5 +1,23 @@
 # News
 
+## 0.1.0 - 2020-02-04
+
+### Improvements
+
+  * Added support for Ruby 2.7.
+    [GitHub#82][GitHub#83][Patch by Yasuo Honda]
+
+  * `Datasets::Hepatitis`: Added.
+    [GitHub#70][Patch by KazuhiroYoshimoto]
+
+  * `Datasets::Downloader`: Added support for query.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * KazuhiroYoshimoto
+
 ## 0.0.9 - 2019-09-09
 
 ### Improvements

data/lib/datasets.rb

@@ -3,6 +3,7 @@ require_relative "datasets/version"
 require_relative "datasets/adult"
 require_relative "datasets/cifar"
 require_relative "datasets/fashion-mnist"
+require_relative "datasets/hepatitis"
 require_relative "datasets/iris"
 require_relative "datasets/libsvm"
 require_relative "datasets/libsvm-dataset-list"

data/lib/datasets/adult.rb

@@ -62,11 +62,12 @@ module Datasets
         data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.#{ext}"
         download(data_path, data_url)
       end
-      CSV.open(data_path,
-               {
+
+      options = {
                  converters: [:numeric, lambda {|f| f.strip}],
                  skip_lines: /\A\|/,
-               }) do |csv|
+      }
+      CSV.open(data_path, **options) do |csv|
         yield(csv)
       end
     end

data/lib/datasets/downloader.rb

@@ -34,7 +34,9 @@ module Datasets
       Net::HTTP.start(@url.hostname,
                       @url.port,
                       :use_ssl => (@url.scheme == "https")) do |http|
-        request = Net::HTTP::Get.new(@url.path, headers)
+        path = @url.path
+        path += "?#{@url.query}" if @url.query
+        request = Net::HTTP::Get.new(path, headers)
         http.request(request) do |response|
           case response
           when Net::HTTPPartialContent

data/lib/datasets/hepatitis.rb

@@ -0,0 +1,207 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class Hepatitis < Dataset
+    class Record < Struct.new(:label,
+                              :age,
+                              :sex,
+                              :steroid,
+                              :antivirals,
+                              :fatigue,
+                              :malaise,
+                              :anorexia,
+                              :liver_big,
+                              :liver_firm,
+                              :spleen_palpable,
+                              :spiders,
+                              :ascites,
+                              :varices,
+                              :bilirubin,
+                              :alkaline_phosphate,
+                              :sgot,
+                              :albumin,
+                              :protime,
+                              :histology)
+      def initialize(*values)
+        super()
+        members.zip(values) do |member, value|
+          __send__("#{member}=", value)
+        end
+      end
+
+      def label=(label)
+        case label
+        when "1"
+          super(:die)
+        when "2"
+          super(:live)
+        else
+          super(label)
+        end
+      end
+
+      def age=(age)
+        super(normalize_integer(age))
+      end
+
+      def sex=(sex)
+        case sex
+        when "1"
+          super(:male)
+        when "2"
+          super(:female)
+        else
+          super(sex)
+        end
+      end
+
+      def steroid=(steroid)
+        super(normalize_boolean(steroid))
+      end
+
+      def antivirals=(antivirals)
+        super(normalize_boolean(antivirals))
+      end
+
+      def fatigue=(fatigue)
+        super(normalize_boolean(fatigue))
+      end
+
+      def malaise=(malaise)
+        super(normalize_boolean(malaise))
+      end
+
+      def anorexia=(anorexia)
+        super(normalize_boolean(anorexia))
+      end
+
+      def liver_big=(liver_big)
+        super(normalize_boolean(liver_big))
+      end
+
+      def liver_firm=(liver_firm)
+        super(normalize_boolean(liver_firm))
+      end
+
+      def spleen_palpable=(spleen_palpable)
+        super(normalize_boolean(spleen_palpable))
+      end
+
+      def spiders=(spiders)
+        super(normalize_boolean(spiders))
+      end
+
+      def ascites=(ascites)
+        super(normalize_boolean(ascites))
+      end
+
+      def varices=(varices)
+        super(normalize_boolean(varices))
+      end
+
+      def bilirubin=(bilirubin)
+        super(normalize_float(bilirubin))
+      end
+
+      def alkaline_phosphate=(alkaline_phosphate)
+        super(normalize_integer(alkaline_phosphate))
+      end
+
+      def sgot=(sgot)
+        super(normalize_integer(sgot))
+      end
+
+      def albumin=(albumin)
+        super(normalize_float(albumin))
+      end
+
+      def protime=(protime)
+        super(normalize_integer(protime))
+      end
+
+      def histology=(histology)
+        super(normalize_boolean(histology))
+      end
+
+      private
+      def normalize_boolean(value)
+        case value
+        when "?"
+          nil
+        when "1"
+          false
+        when "2"
+          true
+        else
+          value
+        end
+      end
+
+      def normalize_float(value)
+        case value
+        when "?"
+          nil
+        else
+          Float(value)
+        end
+      end
+
+      def normalize_integer(value)
+        case value
+        when "?"
+          nil
+        else
+          Integer(value, 10)
+        end
+      end
+    end
+
+    def initialize
+      super()
+      @metadata.id = "hepatitis"
+      @metadata.name = "Hepatitis"
+      @metadata.url = "https://archive.ics.uci.edu/ml/datasets/hepatitis"
+      @metadata.description = lambda do
+        read_names
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def base_url
+      "https://archive.ics.uci.edu/ml/machine-learning-databases/hepatitis"
+    end
+
+    def open_data
+      data_path = cache_dir_path + "hepatitis.csv"
+      unless data_path.exist?
+        data_url = "#{base_url}/hepatitis.data"
+        download(data_path, data_url)
+      end
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def read_names
+      names_path = cache_dir_path + "hepatitis.names"
+      unless names_path.exist?
+        names_url = "#{base_url}/hepatitis.names"
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+  end
+end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.0.9"
+  VERSION = "0.1.0"
 end

data/test/test-hepatitis.rb

@@ -0,0 +1,74 @@
+class HepatitisTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Hepatitis.new
+  end
+
+  def record(*args)
+    Datasets::Hepatitis::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   155,
+                   {
+                     :label => :live,
+                     :age => 30,
+                     :sex => :female,
+                     :steroid => false,
+                     :antivirals => true,
+                     :fatigue => true,
+                     :malaise => true,
+                     :anorexia => true,
+                     :liver_big => false,
+                     :liver_firm => true,
+                     :spleen_palpable => true,
+                     :spiders => true,
+                     :ascites => true,
+                     :varices => true,
+                     :bilirubin => 1.0,
+                     :alkaline_phosphate => 85,
+                     :sgot => 18,
+                     :albumin => 4.0,
+                     :protime => nil,
+                     :histology => false,
+                   },
+                   {
+                     :label => :die,
+                     :age => 43,
+                     :sex => :male,
+                     :steroid => true,
+                     :antivirals => true,
+                     :fatigue => false,
+                     :malaise => true,
+                     :anorexia => true,
+                     :liver_big => true,
+                     :liver_firm => true,
+                     :spleen_palpable => false,
+                     :spiders => false,
+                     :ascites => false,
+                     :varices => true,
+                     :bilirubin => 1.2,
+                     :alkaline_phosphate => 100,
+                     :sgot => 19,
+                     :albumin => 3.1,
+                     :protime => 42,
+                     :histology => true,
+                   }
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h,
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("1. Title: Hepatitis Domain")
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors:
 - tomisuker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-09 00:00:00.000000000 Z
+date: 2020-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -112,7 +112,7 @@ dependencies:
 description: 'You can use datasets easily because you can access each dataset with
   multiple ways such as `#each` and Apache Arrow Record Batch.
 
-'
+  '
 email:
 - tomisuker16@gmail.com
 - kou@clear-code.com
@@ -133,6 +133,7 @@ files:
 - lib/datasets/dictionary.rb
 - lib/datasets/downloader.rb
 - lib/datasets/fashion-mnist.rb
+- lib/datasets/hepatitis.rb
 - lib/datasets/iris.rb
 - lib/datasets/libsvm-dataset-list.rb
 - lib/datasets/libsvm.rb
@@ -152,6 +153,7 @@ files:
 - test/test-cifar.rb
 - test/test-dictionary.rb
 - test/test-fashion-mnist.rb
+- test/test-hepatitis.rb
 - test/test-iris.rb
 - test/test-libsvm-dataset-list.rb
 - test/test-libsvm.rb
@@ -187,19 +189,20 @@ signing_key:
 specification_version: 4
 summary: Red Datasets provides classes that provide common datasets such as iris dataset.
 test_files:
-- test/test-adult.rb
-- test/test-libsvm.rb
+- test/test-wine.rb
+- test/test-iris.rb
 - test/test-wikipedia.rb
-- test/test-libsvm-dataset-list.rb
+- test/test-mnist.rb
 - test/helper.rb
-- test/test-iris.rb
-- test/test-table.rb
-- test/run-test.rb
-- test/test-wine.rb
 - test/test-penn-treebank.rb
-- test/test-postal-code-japan.rb
+- test/run-test.rb
+- test/test-table.rb
+- test/test-fashion-mnist.rb
 - test/test-cifar.rb
-- test/test-mnist.rb
-- test/test-mushroom.rb
 - test/test-dictionary.rb
-- test/test-fashion-mnist.rb
+- test/test-mushroom.rb
+- test/test-libsvm-dataset-list.rb
+- test/test-hepatitis.rb
+- test/test-adult.rb
+- test/test-postal-code-japan.rb
+- test/test-libsvm.rb