red-datasets 0.1.0 → 0.1.1

data/lib/datasets/error.rb

@@ -0,0 +1,4 @@
+module Datasets
+  class Error < StandardError
+  end
+end

data/lib/datasets/mnist.rb

@@ -2,8 +2,6 @@ require 'zlib'
 
 require_relative "dataset"
 
-class SetTypeError < StandardError; end
-
 module Datasets
   class MNIST < Dataset
     BASE_URL = "http://yann.lecun.com/exdb/mnist/"

data/lib/datasets/penguins.rb

@@ -0,0 +1,125 @@
+require_relative "dataset"
+
+module Datasets
+  module PenguinsRawData
+    Record = Struct.new(:study_name,
+                        :sample_number,
+                        :species,
+                        :region,
+                        :island,
+                        :stage,
+                        :individual_id,
+                        :clutch_completion,
+                        :date_egg,
+                        :culmen_length_mm,
+                        :culmen_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :delta_15_n_permil,
+                        :delta_13_c_permil,
+                        :comments)
+
+    class SpeciesBase < Dataset
+      def initialize
+        super
+        species = self.class.name.split("::").last.downcase
+        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.url = self.class::URL
+        @metadata.licenses = ["CC0"]
+        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+      end
+
+      attr_reader :data_path
+
+      def each
+        return to_enum(__method__) unless block_given?
+
+        open_data do |csv|
+          csv.each do |row|
+            next if row[0].nil?
+            record = Record.new(*row.fields)
+            yield record
+          end
+        end
+      end
+
+      private def open_data
+        download unless data_path.exist?
+        CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+          yield csv
+        end
+      end
+
+      private def download
+        super(data_path, metadata.url)
+      end
+    end
+
+    # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
+    class Adelie < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+    end
+
+    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
+    class Gentoo < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+    end
+
+    # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
+    class Chinstrap < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+    end
+  end
+
+  # This dataset provides the same dataset as https://github.com/allisonhorst/palmerpenguins
+  class Penguins < Dataset
+    Record = Struct.new(:species,
+                        :island,
+                        :bill_length_mm,
+                        :bill_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :year)
+
+    def initialize
+      super
+      @metadata.id = "palmerpenguins"
+      @metadata.name = "palmerpenguins"
+      @metadata.url = "https://allisonhorst.github.io/palmerpenguins/"
+      @metadata.licenses = ["CC0"]
+      @metadata.description = "A great dataset for data exploration & visualization, as an alternative to iris"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      species_classes = [
+        PenguinsRawData::Adelie,
+        PenguinsRawData::Gentoo,
+        PenguinsRawData::Chinstrap
+      ]
+
+      species_classes.each do |species_class|
+        species_class.new.each do |raw_record|
+          yield convert_record(raw_record)
+        end
+      end
+    end
+
+    private def convert_record(raw_record)
+      Record.new(raw_record.species.split(' ')[0],
+                 raw_record.island,
+                 raw_record.culmen_length_mm,
+                 raw_record.culmen_depth_mm,
+                 raw_record.flipper_length_mm&.to_i,
+                 raw_record.body_mass_g&.to_i,
+                 raw_record.sex&.downcase,
+                 raw_record.date_egg&.year)
+    end
+  end
+end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/red-datasets.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
 
   spec.add_runtime_dependency("csv", ">= 3.0.5")
+  spec.add_runtime_dependency("rexml")
   spec.add_runtime_dependency("rubyzip")
 
   spec.add_development_dependency("bundler")

data/test/run-test.rb

@@ -13,4 +13,6 @@ $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"
 
+ARGV.unshift("--max-diff-target-string-size=#{10 * 1024}")
+
 exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))

data/test/test-cldr-plurals.rb

@@ -0,0 +1,180 @@
+class CLDRPluralsTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::CLDRPlurals.new
+  end
+
+  def locale(*args)
+    Datasets::CLDRPlurals::Locale.new(*args)
+  end
+
+  def rule(*args)
+    Datasets::CLDRPlurals::Rule.new(*args)
+  end
+
+  test("#each") do
+    locales = @dataset.each.to_a
+    assert_equal([
+                   215,
+                   locale("bm",
+                          [
+                            rule("other",
+                                 nil,
+                                 [
+                                   0..15,
+                                   100,
+                                   1000,
+                                   10000,
+                                   100000,
+                                   1000000,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   0.0..1.5,
+                                   10.0,
+                                   100.0,
+                                   1000.0,
+                                   10000.0,
+                                   100000.0,
+                                   1000000.0,
+                                   :elipsis,
+                                 ])
+                          ]),
+                   locale("kw",
+                          [
+                            rule("zero",
+                                 [:equal, "n", [0]],
+                                 [0],
+                                 [0.0, 0.00, 0.000, 0.0000]),
+                            rule("one",
+                                 [:equal, "n", [1]],
+                                 [1],
+                                 [1.0, 1.00, 1.000, 1.0000]),
+                            rule("two",
+                                 [:or,
+                                  [:equal,
+                                   [:mod, "n", 100],
+                                   [2, 22, 42, 62, 82]],
+                                  [:and,
+                                   [:equal, [:mod, "n", 1000], [0]],
+                                   [:equal,
+                                    [:mod, "n", 100000],
+                                    [1000..20000, 40000, 60000, 80000]]],
+                                  [:and,
+                                   [:not_equal, "n", [0]],
+                                   [:equal, [:mod, "n", 1000000], [100000]]]],
+                                 [
+                                   2,
+                                   22,
+                                   42,
+                                   62,
+                                   82,
+                                   102,
+                                   122,
+                                   142,
+                                   1000,
+                                   10000,
+                                   100000,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   2.0,
+                                   22.0,
+                                   42.0,
+                                   62.0,
+                                   82.0,
+                                   102.0,
+                                   122.0,
+                                   142.0,
+                                   1000.0,
+                                   10000.0,
+                                   100000.0,
+                                   :elipsis,
+                                 ]),
+                            rule("few",
+                                 [:equal,
+                                  [:mod, "n", 100],
+                                  [3, 23, 43, 63, 83]],
+                                 [
+                                   3,
+                                   23,
+                                   43,
+                                   63,
+                                   83,
+                                   103,
+                                   123,
+                                   143,
+                                   1003,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   3.0,
+                                   23.0,
+                                   43.0,
+                                   63.0,
+                                   83.0,
+                                   103.0,
+                                   123.0,
+                                   143.0,
+                                   1003.0,
+                                   :elipsis,
+                                 ]),
+                            rule("many",
+                                 [:and,
+                                  [:not_equal, "n", [1]],
+                                  [:equal,
+                                   [:mod, "n", 100],
+                                   [1, 21, 41, 61, 81]]],
+                                 [
+                                   21,
+                                   41,
+                                   61,
+                                   81,
+                                   101,
+                                   121,
+                                   141,
+                                   161,
+                                   1001,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   21.0,
+                                   41.0,
+                                   61.0,
+                                   81.0,
+                                   101.0,
+                                   121.0,
+                                   141.0,
+                                   161.0,
+                                   1001.0,
+                                   :elipsis,
+                                 ]),
+                            rule("other",
+                                 nil,
+                                 [4..19, 100, 1004, 1000000, :elipsis],
+                                 [
+                                   0.1..0.9,
+                                   1.1..1.7,
+                                   10.0,
+                                   100.0,
+                                   1000.1,
+                                   1000000.0,
+                                   :elipsis,
+                                 ]),
+                          ]),
+                 ],
+                 [
+                   locales.size,
+                   locales[0],
+                   locales[-4],
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("Language plural rules in Unicode Common Locale Data Repository.")
+      end
+    end
+  end
+end

data/test/test-communities.rb

@@ -0,0 +1,290 @@
+class CommunitiesTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Communities.new
+  end
+
+  def record(*args)
+    Datasets::Communities::Record.new(*args)
+  end
+
+  test('#each') do
+    records = @dataset.each.to_a
+    assert_equal([
+                    1994,
+                    {
+                      :state => 8,
+                      :county => nil,
+                      :community => nil,
+                      :community_name => "Lakewoodcity",
+                      :fold => 1,
+                      :population => 0.19,
+                      :household_size => 0.33,
+                      :race_percent_black => 0.02,
+                      :race_percent_white => 0.9,
+                      :race_percent_asian => 0.12,
+                      :race_percent_hispanic => 0.17,
+                      :age_percent_12_to_21 => 0.34,
+                      :age_percent_12_to_29 => 0.47,
+                      :age_percent_16_to_24 => 0.29,
+                      :age_percent_65_and_upper => 0.32,
+                      :n_people_urban => 0.2,
+                      :percent_people_urban => 1,
+                      :median_income => 0.37,
+                      :percent_households_with_wage => 0.72,
+                      :percent_households_with_farm_self => 0.34,
+                      :percent_households_with_investment_income => 0.6,
+                      :percent_households_with_social_security => 0.29,
+                      :percent_households_with_public_assistant => 0.15,
+                      :percent_households_with_retire => 0.43,
+                      :median_family_income => 0.39,
+                      :per_capita_income => 0.4,
+                      :per_capita_income_white => 0.39,
+                      :per_capita_income_black => 0.32,
+                      :per_capita_income_indian => 0.27,
+                      :per_capita_income_asian => 0.27,
+                      :per_capita_income_other => 0.36,
+                      :per_capita_income_hispanic => 0.41,
+                      :n_people_under_poverty => 0.08,
+                      :percent_people_under_poverty => 0.19,
+                      :percent_less_9th_grade => 0.1,
+                      :percent_not_high_school_graduate => 0.18,
+                      :percent_bachelors_or_more => 0.48,
+                      :percent_unemployed => 0.27,
+                      :percent_employed => 0.68,
+                      :percent_employed_manufacturing => 0.23,
+                      :percent_employed_professional_service => 0.41,
+                      :percent_occupations_manufacturing => 0.25,
+                      :percent_occupations_management_professional => 0.52,
+                      :male_percent_divorced => 0.68,
+                      :male_percent_never_married => 0.4,
+                      :female_percent_divorced => 0.75,
+                      :total_percent_divorced => 0.75,
+                      :mean_persons_per_family => 0.35,
+                      :percent_family_2_parents => 0.55,
+                      :percent_kids_2_parents => 0.59,
+                      :percent_young_kids_2_parents => 0.61,
+                      :percent_teen_2_parents => 0.56,
+                      :percent_work_mom_young_kids => 0.74,
+                      :percent_work_mom => 0.76,
+                      :n_illegals => 0.04,
+                      :percent_illegals => 0.14,
+                      :n_immigrants => 0.03,
+                      :percent_immigrants_recent => 0.24,
+                      :percent_immigrants_recent_5 => 0.27,
+                      :percent_immigrants_recent_8 => 0.37,
+                      :percent_immigrants_recent_10 => 0.39,
+                      :percent_population_immigranted_recent => 0.07,
+                      :percent_population_immigranted_recent_5 => 0.07,
+                      :percent_population_immigranted_recent_8 => 0.08,
+                      :percent_population_immigranted_recent_10 => 0.08,
+                      :percent_speak_english_only => 0.89,
+                      :percent_not_speak_english_well => 0.06,
+                      :percent_large_households_family => 0.14,
+                      :percent_large_households_occupied => 0.13,
+                      :mean_persons_per_occupied_household => 0.33,
+                      :mean_persons_per_owner_occupied_household => 0.39,
+                      :mean_persons_per_rental_occupied_household => 0.28,
+                      :percent_persons_owner_occupied_household => 0.55,
+                      :percent_persons_dense_housing => 0.09,
+                      :percent_housing_less_3_bedrooms => 0.51,
+                      :median_n_bedrooms => 0.5,
+                      :n_vacant_households => 0.21,
+                      :percent_housing_occupied => 0.71,
+                      :percent_housing_owner_occupied => 0.52,
+                      :percent_vacant_housing_boarded => 0.05,
+                      :percent_vacant_housing_more_6_months => 0.26,
+                      :median_year_housing_built => 0.65,
+                      :percent_housing_no_phone => 0.14,
+                      :percent_housing_without_full_plumbing => 0.06,
+                      :owner_occupied_housing_lower_quartile => 0.22,
+                      :owner_occupied_housing_median => 0.19,
+                      :owner_occupied_housing_higher_quartile => 0.18,
+                      :rental_housing_lower_quartile => 0.36,
+                      :rental_housing_median => 0.35,
+                      :rental_housing_higher_quartile => 0.38,
+                      :median_rent => 0.34,
+                      :median_rent_percent_household_income => 0.38,
+                      :median_owner_cost_percent_household_income => 0.46,
+                      :median_owner_cost_percent_household_income_no_mortgage => 0.25,
+                      :n_people_shelter => 0.04,
+                      :n_people_street => 0,
+                      :percent_foreign_born => 0.12,
+                      :percent_born_same_state => 0.42,
+                      :percent_same_house_85 => 0.5,
+                      :percent_same_city_85 => 0.51,
+                      :percent_same_state_85 => 0.64,
+                      :lemas_sworn_full_time => 0.03,
+                      :lemas_sworn_full_time_per_population => 0.13,
+                      :lemas_sworn_full_time_field => 0.96,
+                      :lemas_sworn_full_time_field_per_population => 0.17,
+                      :lemas_total_requests => 0.06,
+                      :lemas_total_requests_per_population => 0.18,
+                      :total_requests_per_officer => 0.44,
+                      :n_officers_per_population => 0.13,
+                      :racial_match_community_police => 0.94,
+                      :percent_police_white => 0.93,
+                      :percent_police_black => 0.03,
+                      :percent_police_hispanic => 0.07,
+                      :percent_police_asian => 0.1,
+                      :percent_police_minority => 0.07,
+                      :n_officers_assigned_drug_units => 0.02,
+                      :n_kinds_drugs_seized => 0.57,
+                      :police_average_overtime_worked => 0.29,
+                      :land_area => 0.12,
+                      :population_density => 0.26,
+                      :percent_use_public_transit => 0.2,
+                      :n_police_cars => 0.06,
+                      :n_police_operating_budget => 0.04,
+                      :lemas_percent_police_on_patrol => 0.9,
+                      :lemas_gang_unit_deployed => 0.5,
+                      :lemas_percent_office_drug_units => 0.32,
+                      :police_operating_budget_per_population => 0.14,
+                      :total_violent_crimes_per_population => 0.2,
+                    },
+                    {
+                      :state => 6,
+                      :county => nil,
+                      :community => nil,
+                      :community_name => "Ontariocity",
+                      :fold => 10,
+                      :population => 0.2,
+                      :household_size => 0.78,
+                      :race_percent_black => 0.14,
+                      :race_percent_white => 0.46,
+                      :race_percent_asian => 0.24,
+                      :race_percent_hispanic => 0.77,
+                      :age_percent_12_to_21 => 0.5,
+                      :age_percent_12_to_29 => 0.62,
+                      :age_percent_16_to_24 => 0.4,
+                      :age_percent_65_and_upper => 0.17,
+                      :n_people_urban => 0.21,
+                      :percent_people_urban => 1,
+                      :median_income => 0.4,
+                      :percent_households_with_wage => 0.73,
+                      :percent_households_with_farm_self => 0.22,
+                      :percent_households_with_investment_income => 0.25,
+                      :percent_households_with_social_security => 0.26,
+                      :percent_households_with_public_assistant => 0.47,
+                      :percent_households_with_retire => 0.29,
+                      :median_family_income => 0.36,
+                      :per_capita_income => 0.24,
+                      :per_capita_income_white => 0.28,
+                      :per_capita_income_black => 0.32,
+                      :per_capita_income_indian => 0.22,
+                      :per_capita_income_asian => 0.27,
+                      :per_capita_income_other => 0.25,
+                      :per_capita_income_hispanic => 0.29,
+                      :n_people_under_poverty => 0.16,
+                      :percent_people_under_poverty => 0.35,
+                      :percent_less_9th_grade => 0.5,
+                      :percent_not_high_school_graduate => 0.55,
+                      :percent_bachelors_or_more => 0.16,
+                      :percent_unemployed => 0.47,
+                      :percent_employed => 0.58,
+                      :percent_employed_manufacturing => 0.53,
+                      :percent_employed_professional_service => 0.2,
+                      :percent_occupations_manufacturing => 0.6,
+                      :percent_occupations_management_professional => 0.24,
+                      :male_percent_divorced => 0.49,
+                      :male_percent_never_married => 0.5,
+                      :female_percent_divorced => 0.6,
+                      :total_percent_divorced => 0.57,
+                      :mean_persons_per_family => 0.86,
+                      :percent_family_2_parents => 0.61,
+                      :percent_kids_2_parents => 0.59,
+                      :percent_young_kids_2_parents => 0.64,
+                      :percent_teen_2_parents => 0.6,
+                      :percent_work_mom_young_kids => 0.35,
+                      :percent_work_mom => 0.35,
+                      :n_illegals => 0.11,
+                      :percent_illegals => 0.43,
+                      :n_immigrants => 0.2,
+                      :percent_immigrants_recent => 0.43,
+                      :percent_immigrants_recent_5 => 0.52,
+                      :percent_immigrants_recent_8 => 0.58,
+                      :percent_immigrants_recent_10 => 0.65,
+                      :percent_population_immigranted_recent => 0.68,
+                      :percent_population_immigranted_recent_5 => 0.73,
+                      :percent_population_immigranted_recent_8 => 0.73,
+                      :percent_population_immigranted_recent_10 => 0.75,
+                      :percent_speak_english_only => 0.35,
+                      :percent_not_speak_english_well => 0.77,
+                      :percent_large_households_family => 0.84,
+                      :percent_large_households_occupied => 0.83,
+                      :mean_persons_per_occupied_household => 0.84,
+                      :mean_persons_per_owner_occupied_household => 0.78,
+                      :mean_persons_per_rental_occupied_household => 0.89,
+                      :percent_persons_owner_occupied_household => 0.46,
+                      :percent_persons_dense_housing => 0.76,
+                      :percent_housing_less_3_bedrooms => 0.55,
+                      :median_n_bedrooms => 0,
+                      :n_vacant_households => 0.12,
+                      :percent_housing_occupied => 0.8,
+                      :percent_housing_owner_occupied => 0.49,
+                      :percent_vacant_housing_boarded => 0.1,
+                      :percent_vacant_housing_more_6_months => 0.16,
+                      :median_year_housing_built => 0.71,
+                      :percent_housing_no_phone => 0.32,
+                      :percent_housing_without_full_plumbing => 0.21,
+                      :owner_occupied_housing_lower_quartile => 0.35,
+                      :owner_occupied_housing_median => 0.35,
+                      :owner_occupied_housing_higher_quartile => 0.32,
+                      :rental_housing_lower_quartile => 0.54,
+                      :rental_housing_median => 0.53,
+                      :rental_housing_higher_quartile => 0.61,
+                      :median_rent => 0.54,
+                      :median_rent_percent_household_income => 0.69,
+                      :median_owner_cost_percent_household_income => 0.73,
+                      :median_owner_cost_percent_household_income_no_mortgage => 0.21,
+                      :n_people_shelter => 0.08,
+                      :n_people_street => 0.08,
+                      :percent_foreign_born => 0.68,
+                      :percent_born_same_state => 0.5,
+                      :percent_same_house_85 => 0.34,
+                      :percent_same_city_85 => 0.35,
+                      :percent_same_state_85 => 0.68,
+                      :lemas_sworn_full_time => 0.03,
+                      :lemas_sworn_full_time_per_population => 0.11,
+                      :lemas_sworn_full_time_field => 0.96,
+                      :lemas_sworn_full_time_field_per_population => 0.14,
+                      :lemas_total_requests => 0.04,
+                      :lemas_total_requests_per_population => 0.11,
+                      :total_requests_per_officer => 0.3,
+                      :n_officers_per_population => 0.11,
+                      :racial_match_community_police => 0.45,
+                      :percent_police_white => 0.74,
+                      :percent_police_black => 0.34,
+                      :percent_police_hispanic => 0.07,
+                      :percent_police_asian => 0,
+                      :percent_police_minority => 0.28,
+                      :n_officers_assigned_drug_units => 0.07,
+                      :n_kinds_drugs_seized => 0.36,
+                      :police_average_overtime_worked => 0.11,
+                      :land_area => 0.11,
+                      :population_density => 0.3,
+                      :percent_use_public_transit => 0.05,
+                      :n_police_cars => 0.08,
+                      :n_police_operating_budget => 0.04,
+                      :lemas_percent_police_on_patrol => 0.73,
+                      :lemas_gang_unit_deployed => 0.5,
+                      :lemas_percent_office_drug_units => 1,
+                      :police_operating_budget_per_population => 0.13,
+                      :total_violent_crimes_per_population => 0.48
+                    },
+                 ],
+                 [
+                    records.size,
+                    records[0].to_h,
+                    records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("Title: Communities and Crime")
+      end
+    end
+  end
+end