red-datasets 0.1.7 → 0.1.9

data/lib/datasets/nagoya-university-conversation-corpus.rb

@@ -28,8 +28,8 @@ module Datasets
 
     def initialize
       super()
-      @metadata.id = 'nagoya-university-conversation-curpus'
-      @metadata.name = 'Nagoya University Conversation Curpus'
+      @metadata.id = 'nagoya-university-conversation-corpus'
+      @metadata.name = 'Nagoya University Conversation Corpus'
       @metadata.url = 'https://mmsrv.ninjal.ac.jp/nucc/'
       @metadata.licenses = ['CC-BY-NC-ND-4.0']
       @metadata.description = <<~DESCRIPTION

data/lib/datasets/penguins.rb

@@ -26,7 +26,9 @@ module Datasets
         super
         species = self.class.name.split("::").last.downcase
         @metadata.id = "palmerpenguins-#{species}"
-        @metadata.url = self.class::URL
+        package_id = http_parameters["packageid"]
+        @metadata.url = "https://portal.edirepository.org/nis/mapbrowse" +
+                        "?packageid=#{package_id}"
         @metadata.licenses = ["CC0-1.0"]
         @data_path = cache_dir_path + "#{species}.csv"
       end
@@ -46,7 +48,10 @@ module Datasets
       end
 
       private def open_data
-        download(data_path, metadata.url)
+        download(data_path,
+                 "https://portal.edirepository.org/nis/dataviewer",
+                 http_method: :post,
+                 http_parameters: http_parameters)
         CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
           yield csv
         end
@@ -56,19 +61,37 @@ module Datasets
     # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
     class Adelie < SpeciesBase
       DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.219.3",
+          "entityid" => "002f3893385f710df69eeebe893144ff",
+        }
+      end
     end
 
     # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
     class Chinstrap < SpeciesBase
       DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.221.2",
+          "entityid" => "fe853aa8f7a59aa84cdd3197619ef462",
+        }
+      end
     end
 
     # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
     class Gentoo < SpeciesBase
       DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.220.3",
+          "entityid" => "e03b43c924f226486f2f0ab6709d2381",
+        }
+      end
     end
   end
 

data/lib/datasets/postal-code-japan.rb

@@ -41,7 +41,7 @@ module Datasets
       super()
       @reading = reading
       unless VALID_READINGS.include?(@reading)
-        message = ":reading must be one of ["
+        message = +":reading must be one of ["
         message << VALID_READINGS.collect(&:inspect).join(", ")
         message << "]: #{@reading.inspect}"
         raise ArgumentError, message
@@ -104,14 +104,14 @@ module Datasets
 
     private
     def open_data
-      data_url = "https://www.post.japanpost.jp/zipcode/dl"
+      data_url = +"https://www.post.japanpost.jp/zipcode/dl"
       case @reading
       when :lowercase
         data_url << "/kogaki/zip/ken_all.zip"
       when :uppercase
         data_url << "/oogaki/zip/ken_all.zip"
       when :romaji
-        data_url << "/roman/ken_all_rome.zip"
+        data_url << "/roman/KEN_ALL_ROME.zip"
       end
       data_path = cache_dir_path + "#{@reading}-ken-all.zip"
       download(data_path, data_url)

data/lib/datasets/quora-duplicate-question-pair.rb

@@ -43,7 +43,7 @@ module Datasets
       data_path = cache_dir_path + "quora_duplicate_questions.tsv"
       data_url = "https://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
       download(data_path, data_url)
-      CSV.open(data_path, col_sep: "\t", headers: true, converters: :all) do |csv|
+      CSV.open(data_path, col_sep: "\t", headers: true, converters: :integer) do |csv|
         yield(csv)
       end
     end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.7"
+  VERSION = "0.1.9"
 end

data/lib/datasets/wikipedia-kyoto-japanese-english.rb

@@ -126,7 +126,7 @@ articles (related to Kyoto) into English.
         @sentence = nil
         @text_container_stack = []
         @element_stack = []
-        @text_stack = [""]
+        @text_stack = [+""]
       end
 
       def tag_start(name, attributes)
@@ -207,7 +207,7 @@ articles (related to Kyoto) into English.
       private
       def push_stacks(name, attributes)
         @element_stack.push({name: name, attributes: attributes})
-        @text_stack.push("")
+        @text_stack.push(+"")
       end
 
       def pop_stacks

data/lib/datasets/wikipedia.rb

@@ -90,7 +90,7 @@ module Datasets
         @contributor = nil
         @current_tag = nil
         @tag_stack = []
-        @text_stack = [""]
+        @text_stack = [+""]
         @first_page = true
       end
 
@@ -172,7 +172,7 @@ module Datasets
 
       def push_stacks(tag)
         @tag_stack << tag
-        @text_stack << ""
+        @text_stack << +""
       end
 
       def pop_stacks

data/test/japanese-date-parser-test.rb

@@ -0,0 +1,27 @@
+class JapaneseDateParserTest < Test::Unit::TestCase
+  def setup
+    @parser = Datasets::JapaneseDateParser.new
+  end
+
+  data("month and day with leading a space in Heisei", ["H10.01.01", "平成10年 1月 1日"])
+  data("month         with leading a space in Heisei", ["H10.01.10", "平成10年 1月10日"])
+  data("          day with leading a space in Heisei", ["H10.10.01", "平成10年10月 1日"])
+  data("           without leading a space in Heisei", ["H10.10.10", "平成10年10月10日"])
+  data("year, month and day with leading a space in Reiwa", ["R02.01.01", "令和 2年 1月 1日"])
+  data("year, month         with leading a space in Reiwa", ["R02.01.10", "令和 2年 1月10日"])
+  data("year,           day with leading a space in Reiwa", ["R02.10.01", "令和 2年10月 1日"])
+  data("year,            without leading a space in Reiwa", ["R02.10.10", "令和 2年10月10日"])
+  data("boundary within Heisei", ["H31.04.30", "平成31年 4月30日"])
+  data("boundary within Reiwa", ["R01.05.01", "令和元年 5月 1日"])
+  test("#parse") do
+    expected_jisx0301, japanese_date_string = data
+    assert_equal(expected_jisx0301, @parser.parse(japanese_date_string).jisx0301)
+  end
+
+  test("unsupported era initial range") do
+    expected_message = "era must be one of [平成, 令和]: 昭和"
+    assert_raise(Datasets::JapaneseDateParser::UnsupportedEraInitialRange.new(expected_message)) do
+      @parser.parse("昭和元年 1月 1日")
+    end
+  end
+end

data/test/test-adult.rb

@@ -9,49 +9,24 @@ class AdultTest < Test::Unit::TestCase
     end
 
     test("#each") do
-      records = @dataset.each.to_a
-      assert_equal([
-                     32561,
-                     {
-                       :age => 39,
-                       :work_class => "State-gov",
-                       :final_weight => 77516,
-                       :education => "Bachelors",
-                       :n_education_years => 13,
-                       :marital_status => "Never-married",
-                       :occupation => "Adm-clerical",
-                       :relationship => "Not-in-family",
-                       :race => "White",
-                       :sex => "Male",
-                       :capital_gain => 2174,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => "<=50K"
-                     },
-                     {
-                       :age => 52,
-                       :work_class => "Self-emp-inc",
-                       :final_weight => 287927,
-                       :education => "HS-grad",
-                       :n_education_years => 9,
-                       :marital_status => "Married-civ-spouse",
-                       :occupation => "Exec-managerial",
-                       :relationship => "Wife",
-                       :race => "White",
-                       :sex => "Female",
-                       :capital_gain => 15024,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => ">50K"
-                     }
-                   ],
-                   [
-                     records.size,
-                     records[0].to_h,
-                     records[-1].to_h
-                   ])
+      assert_equal({
+                     :age => 39,
+                     :work_class => "State-gov",
+                     :final_weight => 77516,
+                     :education => "Bachelors",
+                     :n_education_years => 13,
+                     :marital_status => "Never-married",
+                     :occupation => "Adm-clerical",
+                     :relationship => "Not-in-family",
+                     :race => "White",
+                     :sex => "Male",
+                     :capital_gain => 2174,
+                     :capital_loss => 0,
+                     :hours_per_week => 40,
+                     :native_country => "United-States",
+                     :label => "<=50K"
+                   },
+                   @dataset.each.next.to_h)
     end
   end
 
@@ -65,49 +40,24 @@ class AdultTest < Test::Unit::TestCase
     end
 
     test("#each") do
-      records = @dataset.each.to_a
-      assert_equal([
-                     16281,
-                     {
-                       :age => 25,
-                       :work_class => "Private",
-                       :final_weight => 226802,
-                       :education => "11th",
-                       :n_education_years => 7,
-                       :marital_status => "Never-married",
-                       :occupation => "Machine-op-inspct",
-                       :relationship => "Own-child",
-                       :race => "Black",
-                       :sex => "Male",
-                       :capital_gain => 0,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => "<=50K."
-                     },
-                     {
-                       :age => 35,
-                       :work_class => "Self-emp-inc",
-                       :final_weight => 182148,
-                       :education => "Bachelors",
-                       :n_education_years => 13,
-                       :marital_status => "Married-civ-spouse",
-                       :occupation => "Exec-managerial",
-                       :relationship => "Husband",
-                       :race => "White",
-                       :sex => "Male",
-                       :capital_gain => 0,
-                       :capital_loss => 0,
-                       :hours_per_week => 60,
-                       :native_country => "United-States",
-                       :label => ">50K."
-                     }
-                   ],
-                   [
-                     records.size,
-                     records[0].to_h,
-                     records[-1].to_h
-                   ])
+      assert_equal({
+                     :age => 25,
+                     :work_class => "Private",
+                     :final_weight => 226802,
+                     :education => "11th",
+                     :n_education_years => 7,
+                     :marital_status => "Never-married",
+                     :occupation => "Machine-op-inspct",
+                     :relationship => "Own-child",
+                     :race => "Black",
+                     :sex => "Male",
+                     :capital_gain => 0,
+                     :capital_loss => 0,
+                     :hours_per_week => 40,
+                     :native_country => "United-States",
+                     :label => "<=50K.",
+                   },
+                   @dataset.each.next.to_h)
     end
   end
 

data/test/test-aozora-bunko.rb

@@ -114,13 +114,13 @@ class AozoraBunkoTest < Test::Unit::TestCase
           book = Datasets::AozoraBunko::Book.new
           book.cache_path = @cache_path
 
-          book.title_id = '000750'
-          book.person_id = '000146'
-          book.html_file_url = 'http://www.lcv.ne.jp/~ibs52086/fire/'
+          book.title_id = '061551'
+          book.person_id = '002225'
+          book.html_file_url = 'http://minken.party/2019/03/09/yakunin/'
           book.html_file_character_encoding = 'UTF-8'
 
-          assert_equal('<title>種田山頭火句集 | 『草木塔抄』他　FIRE ON THE MOUNTAIN</title>',
-                       book.html.split("\n")[7])
+          assert_equal('<title>自由民権現代研究会</title>',
+                       book.html[/<title>.*?<\/title>/])
         end
       end
 

data/test/test-california-housing.rb

@@ -8,37 +8,18 @@ class CaliforniaHousingTest < Test::Unit::TestCase
   end
 
   test("#each") do
-    records = @dataset.each.to_a
-    assert_equal([
-                   20640,
-                   {
-                     median_house_value: 452600.000000,
-                     median_income: 8.325200,
-                     housing_median_age: 41.000000,
-                     total_rooms: 880.000000,
-                     total_bedrooms: 129.000000,
-                     population: 322.000000,
-                     households: 126.000000,
-                     latitude: 37.880000,
-                     longitude: -122.230000
-                   },
-                   {
-                     median_house_value: 89400.000000,
-                     median_income: 2.388600,
-                     housing_median_age: 16.000000,
-                     total_rooms: 2785.000000,
-                     total_bedrooms: 616.000000,
-                     population: 1387.000000,
-                     households: 530.000000,
-                     latitude: 39.370000,
-                     longitude: -121.240000
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h
-                 ])
+    assert_equal({
+                   median_house_value: 452600.000000,
+                   median_income: 8.325200,
+                   housing_median_age: 41.000000,
+                   total_rooms: 880.000000,
+                   total_bedrooms: 129.000000,
+                   population: 322.000000,
+                   households: 126.000000,
+                   latitude: 37.880000,
+                   longitude: -122.230000,
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do

data/test/test-cldr-plurals.rb

@@ -14,7 +14,7 @@ class CLDRPluralsTest < Test::Unit::TestCase
   test("#each") do
     locales = @dataset.each.to_a
     assert_equal([
-                   219,
+                   220,
                    locale("bm",
                           [
                             rule("other",

data/test/test-diamonds.rb

@@ -8,39 +8,19 @@ class DiamondsTest < Test::Unit::TestCase
   end
 
   test("#each") do
-    records = @dataset.each.to_a
-    assert_equal([
-                   53940,
-                   {
-                     carat: 0.23,
-                     clarity: "SI2",
-                     color: "E",
-                     cut: "Ideal",
-                     depth: 61.5,
-                     price: 326,
-                     table: 55.0,
-                     x: 3.95,
-                     y: 3.98,
-                     z: 2.43,
-                   },
-                   {
-                     carat: 0.75,
-                     clarity: "SI2",
-                     color: "D",
-                     cut: "Ideal",
-                     depth: 62.2,
-                     price: 2757,
-                     table: 55.0,
-                     x: 5.83,
-                     y: 5.87,
-                     z: 3.64,
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h
-                 ])
+    assert_equal({
+                   carat: 0.23,
+                   clarity: "SI2",
+                   color: "E",
+                   cut: "Ideal",
+                   depth: 61.5,
+                   price: 326,
+                   table: 55.0,
+                   x: 3.95,
+                   y: 3.98,
+                   z: 2.43,
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do

data/test/test-downloader.rb

@@ -17,7 +17,7 @@ class DownloaderTest < Test::Unit::TestCase
       output_path = @tmp_dir + "file"
       downloader = Datasets::Downloader.new(first_url)
 
-      downloader.define_singleton_method(:start_http) do |url, headers|
+      downloader.define_singleton_method(:start_http) do |url, fallback_urls, headers|
         raise Datasets::Downloader::TooManyRedirects, "too many redirections: #{last_url}"
       end
 

data/test/test-geolonia.rb

@@ -4,47 +4,23 @@ class GeoloniaTest < Test::Unit::TestCase
   end
 
   test('#each') do
-    records = @dataset.each.to_a
-    assert_equal([
-                   277616,
-                   {
-                     :prefecture_code => "01",
-                     :prefecture_name => "北海道",
-                     :prefecture_kana => "ホッカイドウ",
-                     :prefecture_romaji => "HOKKAIDO",
-                     :municipality_code => "01101",
-                     :municipality_name => "札幌市中央区",
-                     :municipality_kana => "サッポロシチュウオウク",
-                     :municipality_romaji => "SAPPORO SHI CHUO KU",
-                     :street_name => "旭ケ丘一丁目",
-                     :street_kana => "アサヒガオカ 1",
-                     :street_romaji => "ASAHIGAOKA 1",
-                     :alias => nil,
-                     :latitude => "43.04223",
-                     :longitude => "141.319722"
-                   },
-                   {
-                     :prefecture_code => "47",
-                     :prefecture_name => "沖縄県",
-                     :prefecture_kana => "オキナワケン",
-                     :prefecture_romaji => "OKINAWA KEN",
-                     :municipality_code => "47382",
-                     :municipality_name => "八重山郡与那国町",
-                     :municipality_kana => "ヤエヤマグンヨナグニチョウ",
-                     :municipality_romaji => "YAEYAMA GUN YONAGUNI CHO",
-                     :street_name => "字与那国",
-                     :street_kana => nil,
-                     :street_romaji => nil,
-                     :alias => nil,
-                     :latitude => "24.455925",
-                     :longitude => "122.987678",
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h,
-                 ])
+    assert_equal({
+                   :prefecture_code => "01",
+                   :prefecture_name => "北海道",
+                   :prefecture_kana => "ホッカイドウ",
+                   :prefecture_romaji => "HOKKAIDO",
+                   :municipality_code => "01101",
+                   :municipality_name => "札幌市中央区",
+                   :municipality_kana => "サッポロシチュウオウク",
+                   :municipality_romaji => "SAPPORO SHI CHUO KU",
+                   :street_name => "旭ケ丘一丁目",
+                   :street_kana => "アサヒガオカ 1",
+                   :street_romaji => "ASAHIGAOKA 1",
+                   :alias => nil,
+                   :latitude => "43.04223",
+                   :longitude => "141.319722",
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do