red-datasets 0.0.7 → 0.1.2

data/test/test-hepatitis.rb

@@ -0,0 +1,74 @@
+class HepatitisTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Hepatitis.new
+  end
+
+  def record(*args)
+    Datasets::Hepatitis::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   155,
+                   {
+                     :label => :live,
+                     :age => 30,
+                     :sex => :female,
+                     :steroid => false,
+                     :antivirals => true,
+                     :fatigue => true,
+                     :malaise => true,
+                     :anorexia => true,
+                     :liver_big => false,
+                     :liver_firm => true,
+                     :spleen_palpable => true,
+                     :spiders => true,
+                     :ascites => true,
+                     :varices => true,
+                     :bilirubin => 1.0,
+                     :alkaline_phosphate => 85,
+                     :sgot => 18,
+                     :albumin => 4.0,
+                     :protime => nil,
+                     :histology => false,
+                   },
+                   {
+                     :label => :die,
+                     :age => 43,
+                     :sex => :male,
+                     :steroid => true,
+                     :antivirals => true,
+                     :fatigue => false,
+                     :malaise => true,
+                     :anorexia => true,
+                     :liver_big => true,
+                     :liver_firm => true,
+                     :spleen_palpable => false,
+                     :spiders => false,
+                     :ascites => false,
+                     :varices => true,
+                     :bilirubin => 1.2,
+                     :alkaline_phosphate => 100,
+                     :sgot => 19,
+                     :albumin => 3.1,
+                     :protime => 42,
+                     :histology => true,
+                   }
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h,
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("1. Title: Hepatitis Domain")
+      end
+    end
+  end
+end

data/test/test-libsvm-dataset-list.rb

@@ -0,0 +1,47 @@
+class LIBSVMDatasetListTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::LIBSVMDatasetList.new
+  end
+
+  test("#each") do
+    assert_equal({
+                   name: "a1a",
+                   source: "UCI / Adult",
+                   preprocessing:
+                     "The original Adult data set has 14 features, " +
+                     "among which six are continuous and eight are " +
+                     "categorical. In this data set, continuous features " +
+                     "are discretized into quantiles, and each quantile is " +
+                     "represented by a binary feature. Also, a categorical " +
+                     "feature with m categories is converted to m binary " +
+                     "features. Details on how each feature is converted " +
+                     "can be found in the beginning of each file from this " +
+                     "page. [JP98a]",
+                   n_classes: 2,
+                   n_data: 1605,
+                   n_features: 123,
+                   files: [
+                     {
+                       name: "a1a",
+                       url: "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/a1a",
+                       note: nil,
+                     },
+                     {
+                       name: "a1a.t",
+                       url: "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/a1a.t",
+                       note: "testing",
+                     }
+                   ],
+                 },
+                 @dataset.first.to_h)
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("This page contains many classification, ")
+      end
+    end
+  end
+end

data/test/test-libsvm.rb

@@ -0,0 +1,205 @@
+class LIBSVMDatasetTest < Test::Unit::TestCase
+  test(":note") do
+    dataset = Datasets::LIBSVM.new("a1a", note: "testing")
+    hash = {label: -1}
+    n_features = 123
+    n_features.times do |i|
+      hash[i] = 0
+    end
+    [5, 7, 14, 19, 39, 40, 51, 63, 67, 73, 74, 76, 78, 83].each do |i|
+      hash[i - 1] = 1
+    end
+    assert_equal(hash,
+                 dataset.first.to_h)
+  end
+
+  test(":default_feature_value") do
+    dataset = Datasets::LIBSVM.new("a1a", default_feature_value: nil)
+    hash = {label: -1}
+    n_features = 123
+    n_features.times do |i|
+      hash[i] = nil
+    end
+    [3, 11, 14, 19, 39, 42, 55, 64, 67, 73, 75, 76, 80, 83].each do |i|
+      hash[i - 1] = 1
+    end
+    assert_equal(hash,
+                 dataset.first.to_h)
+  end
+
+  test("classification") do
+    dataset = Datasets::LIBSVM.new("a1a")
+    hash = {label: -1}
+    n_features = 123
+    n_features.times do |i|
+      hash[i] = 0
+    end
+    [3, 11, 14, 19, 39, 42, 55, 64, 67, 73, 75, 76, 80, 83].each do |i|
+      hash[i - 1] = 1
+    end
+    assert_equal(hash,
+                 dataset.first.to_h)
+  end
+
+  test("regression") do
+    dataset = Datasets::LIBSVM.new("abalone")
+    hash = {label: 15}
+    n_features = 8
+    n_features.times do |i|
+      hash[i] = 0
+    end
+    [
+      [1, 1],
+      [2, 0.455],
+      [3, 0.365],
+      [4, 0.095],
+      [5, 0.514],
+      [6, 0.2245],
+      [7, 0.101],
+      [8, 0.15],
+    ].each do |i, value|
+      hash[i - 1] = value
+    end
+    assert_equal(hash,
+                 dataset.first.to_h)
+  end
+
+  test("multi-label") do
+    dataset = Datasets::LIBSVM.new("mediamill (exp1)")
+    hash = {label: [65, 67, 11, 31]}
+    n_features = 120
+    n_features.times do |i|
+      hash[i] = 0
+    end
+    [
+      [1, 0.380877],
+      [2, 0.494079],
+      [3, 0.540009],
+      [4, 0.422926],
+      [5, 0.158318],
+      [6, 0.326975],
+      [7, 0.390861],
+      [8, 0.527121],
+      [9, 0.254052],
+      [10, 0.223731],
+      [11, 0.040285],
+      [12, 0.141133],
+      [13, 0.112249],
+      [14, 0.263171],
+      [15, 0.147020],
+      [16, 0.472414],
+      [17, 0.592614],
+      [18, 0.653138],
+      [19, 0.499867],
+      [20, 0.196520],
+      [21, 0.403892],
+      [22, 0.482395],
+      [23, 0.619219],
+      [24, 0.320346],
+      [25, 0.281251],
+      [26, 0.054750],
+      [27, 0.180459],
+      [28, 0.139964],
+      [29, 0.319925],
+      [30, 0.181216],
+      [31, 0.364294],
+      [32, 0.407211],
+      [33, 0.368926],
+      [34, 0.427661],
+      [35, 0.211391],
+      [36, 0.364345],
+      [37, 0.370710],
+      [38, 0.409107],
+      [39, 0.289299],
+      [40, 0.243053],
+      [41, 0.063121],
+      [42, 0.193587],
+      [43, 0.158755],
+      [44, 0.316054],
+      [45, 0.197410],
+      [46, 0.656168],
+      [47, 0.678760],
+      [48, 0.650831],
+      [49, 0.674636],
+      [50, 0.492428],
+      [51, 0.623887],
+      [52, 0.610622],
+      [53, 0.678219],
+      [54, 0.574774],
+      [55, 0.523073],
+      [56, 0.206804],
+      [57, 0.496294],
+      [58, 0.429221],
+      [59, 0.586611],
+      [60, 0.471550],
+      [61, 0.284480],
+      [62, 0.432466],
+      [63, 0.498075],
+      [64, 0.408141],
+      [65, 0.102713],
+      [66, 0.303028],
+      [67, 0.309501],
+      [68, 0.444855],
+      [69, 0.191727],
+      [70, 0.174895],
+      [71, 0.034143],
+      [72, 0.153099],
+      [73, 0.068318],
+      [74, 0.217020],
+      [75, 0.099688],
+      [76, 0.409862],
+      [77, 0.561918],
+      [78, 0.612031],
+      [79, 0.514471],
+      [80, 0.146015],
+      [81, 0.398807],
+      [82, 0.383295],
+      [83, 0.548485],
+      [84, 0.282937],
+      [85, 0.252712],
+      [86, 0.051008],
+      [87, 0.223110],
+      [88, 0.098112],
+      [89, 0.299672],
+      [90, 0.144873],
+      [91, 0.308488],
+      [92, 0.358478],
+      [93, 0.352077],
+      [94, 0.394686],
+      [95, 0.157513],
+      [96, 0.339370],
+      [97, 0.321558],
+      [98, 0.341373],
+      [99, 0.247969],
+      [100, 0.206070],
+      [101, 0.061001],
+      [102, 0.216793],
+      [103, 0.112389],
+      [104, 0.273648],
+      [105, 0.152745],
+      [106, 0.598081],
+      [107, 0.621687],
+      [108, 0.607213],
+      [109, 0.644025],
+      [110, 0.394948],
+      [111, 0.593651],
+      [112, 0.551529],
+      [113, 0.574392],
+      [114, 0.511032],
+      [115, 0.463997],
+      [116, 0.202034],
+      [117, 0.492341],
+      [118, 0.317983],
+      [119, 0.547807],
+      [120, 0.393778],
+    ].each do |i, value|
+      hash[i - 1] = value
+    end
+    assert_equal(hash,
+                 dataset.first.to_h)
+  end
+
+  test("string") do
+    # TODO
+  end
+end

data/test/test-mushroom.rb

@@ -0,0 +1,80 @@
+class MushroomTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Mushroom.new
+  end
+
+  def record(*args)
+    Datasets::Mushroom::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   8124,
+                   {
+                     :label => "poisonous",
+                     :cap_shape => "convex",
+                     :cap_surface => "smooth",
+                     :cap_color => "brown",
+                     :bruises => "bruises",
+                     :odor => "pungent",
+                     :gill_attachment => "free",
+                     :gill_spacing => "close",
+                     :gill_size => "narrow",
+                     :gill_color => "black",
+                     :stalk_shape => "enlarging",
+                     :stalk_root => "equal",
+                     :stalk_surface_above_ring => "smooth",
+                     :stalk_surface_below_ring => "smooth",
+                     :stalk_color_above_ring => "white",
+                     :stalk_color_below_ring => "white",
+                     :veil_type => "partial",
+                     :veil_color => "white",
+                     :n_rings => 1,
+                     :ring_type => "pendant",
+                     :spore_print_color => "black",
+                     :population => "scattered",
+                     :habitat => "urban"
+                   },
+                   {
+                     :label => "edible",
+                     :cap_shape => "convex",
+                     :cap_surface => "smooth",
+                     :cap_color => "brown",
+                     :bruises => "no",
+                     :odor => "none",
+                     :gill_attachment => "attached",
+                     :gill_spacing => "close",
+                     :gill_size => "broad",
+                     :gill_color => "yellow",
+                     :stalk_shape => "enlarging",
+                     :stalk_root => "missing",
+                     :stalk_surface_above_ring => "smooth",
+                     :stalk_surface_below_ring => "smooth",
+                     :stalk_color_above_ring => "orange",
+                     :stalk_color_below_ring => "orange",
+                     :veil_type => "partial",
+                     :veil_color => "orange",
+                     :n_rings => 1,
+                     :ring_type => "pendant",
+                     :spore_print_color => "orange",
+                     :population => "clustered",
+                     :habitat => "leaves"
+                   }
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("1. Title: Mushroom Database")
+      end
+    end
+  end
+end

data/test/test-penguins.rb

@@ -0,0 +1,251 @@
+class PenguinsTest < Test::Unit::TestCase
+  sub_test_case("PenguinsRawData::SpeciesBase") do
+    test("#data_path") do
+      data_paths = [ Datasets::PenguinsRawData::Adelie,
+                     Datasets::PenguinsRawData::Gentoo,
+                     Datasets::PenguinsRawData::Chinstrap ].map {|cls|
+                       dataset = cls.new
+                       dataset.data_path.relative_path_from(dataset.send(:cache_dir_path)).to_s
+                     }
+      assert_equal(["penguins/adelie.csv", "penguins/gentoo.csv", "penguins/chinstrap.csv"],
+                   data_paths)
+    end
+  end
+
+  sub_test_case("Adelie") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Adelie.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 152,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Adelie Penguin (Pygoscelis adeliae)",
+                       region: "Anvers",
+                       island: "Torgersen",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N1A1",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2007, 11, 11),
+                       culmen_length_mm: 39.1,
+                       culmen_depth_mm: 18.7,
+                       flipper_length_mm: 181,
+                       body_mass_g: 3750,
+                       sex: "MALE",
+                       delta_15_n_permil: nil,
+                       delta_13_c_permil: nil,
+                       comments: "Not enough blood for isotopes."
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number: 152,
+                       species: "Adelie Penguin (Pygoscelis adeliae)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N85A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 17),
+                       culmen_length_mm: 41.5,
+                       culmen_depth_mm: 18.5,
+                       flipper_length_mm: 201,
+                       body_mass_g: 4000,
+                       sex: "MALE",
+                       delta_15_n_permil: 8.89640,
+                       delta_13_c_permil: -26.06967,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Gentoo") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Gentoo.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 124,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Gentoo penguin (Pygoscelis papua)",
+                       region: "Anvers",
+                       island: "Biscoe",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N31A1",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2007, 11, 27),
+                       culmen_length_mm: 46.1,
+                       culmen_depth_mm: 13.2,
+                       flipper_length_mm: 211,
+                       body_mass_g: 4500,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 7.993,
+                       delta_13_c_permil: -25.5139,
+                       comments: nil
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number:  124,
+                       species: "Gentoo penguin (Pygoscelis papua)",
+                       region: "Anvers",
+                       island: "Biscoe",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N43A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 22),
+                       culmen_length_mm: 49.9,
+                       culmen_depth_mm: 16.1,
+                       flipper_length_mm: 213,
+                       body_mass_g: 5400,
+                       sex: "MALE",
+                       delta_15_n_permil: 8.3639,
+                       delta_13_c_permil: -26.15531,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Chinstrap") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Chinstrap.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 68,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Chinstrap penguin (Pygoscelis antarctica)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N61A1",
+                       clutch_completion: "No",
+                       date_egg: DateTime.new(2007, 11, 19),
+                       culmen_length_mm: 46.5,
+                       culmen_depth_mm: 17.9,
+                       flipper_length_mm: 192,
+                       body_mass_g: 3500,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 9.03935,
+                       delta_13_c_permil: -24.30229,
+                       comments: "Nest never observed with full clutch."
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number: 68,
+                       species: "Chinstrap penguin (Pygoscelis antarctica)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N100A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 21),
+                       culmen_length_mm: 50.2,
+                       culmen_depth_mm: 18.7,
+                       flipper_length_mm: 198,
+                       body_mass_g: 3775,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 9.39305,
+                       delta_13_c_permil: -24.25255,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Penguins") do
+    def setup
+      @dataset = Datasets::Penguins.new
+    end
+
+    test("order of species") do
+      species_values = @dataset.map {|r| r.species }.uniq
+      assert_equal(["Adelie", "Chinstrap", "Gentoo"],
+                   species_values)
+    end
+
+    test("data cleansing") do
+      sex_values = @dataset.map {|r| r.sex }.uniq.compact.sort
+      assert_equal(["female", "male"],
+                   sex_values)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     344,
+                     {
+                       species: "Adelie",
+                       island: "Torgersen",
+                       bill_length_mm: 39.1,
+                       bill_depth_mm: 18.7,
+                       flipper_length_mm: 181,
+                       body_mass_g: 3750,
+                       sex: "male",
+                       year: 2007
+                     },
+                     {
+                       species: "Chinstrap",
+                       island: "Dream",
+                       bill_length_mm: 46.5,
+                       bill_depth_mm: 17.9,
+                       flipper_length_mm: 192,
+                       body_mass_g: 3500,
+                       sex: "female",
+                       year: 2007
+                     },
+                     {
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 46.1,
+                       bill_depth_mm: 13.2,
+                       flipper_length_mm: 211,
+                       body_mass_g: 4500,
+                       sex: "female",
+                       year: 2007
+                     },
+                     {
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 49.9,
+                       bill_depth_mm: 16.1,
+                       flipper_length_mm: 213,
+                       body_mass_g: 5400,
+                       sex: "male",
+                       year: 2009
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[152].to_h,
+                     records[220].to_h,
+                     records[-1].to_h,
+                   ])
+    end
+  end
+end