red-datasets 0.1.7 → 0.1.8

data/lib/datasets/quora-duplicate-question-pair.rb

@@ -43,7 +43,7 @@ module Datasets
       data_path = cache_dir_path + "quora_duplicate_questions.tsv"
       data_url = "https://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
       download(data_path, data_url)
-      CSV.open(data_path, col_sep: "\t", headers: true, converters: :all) do |csv|
+      CSV.open(data_path, col_sep: "\t", headers: true, converters: :integer) do |csv|
         yield(csv)
       end
     end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

data/lib/datasets/wikipedia-kyoto-japanese-english.rb

@@ -126,7 +126,7 @@ articles (related to Kyoto) into English.
         @sentence = nil
         @text_container_stack = []
         @element_stack = []
-        @text_stack = [""]
+        @text_stack = [+""]
       end
 
       def tag_start(name, attributes)
@@ -207,7 +207,7 @@ articles (related to Kyoto) into English.
       private
       def push_stacks(name, attributes)
         @element_stack.push({name: name, attributes: attributes})
-        @text_stack.push("")
+        @text_stack.push(+"")
       end
 
       def pop_stacks

data/lib/datasets/wikipedia.rb

@@ -90,7 +90,7 @@ module Datasets
         @contributor = nil
         @current_tag = nil
         @tag_stack = []
-        @text_stack = [""]
+        @text_stack = [+""]
         @first_page = true
       end
 
@@ -172,7 +172,7 @@ module Datasets
 
       def push_stacks(tag)
         @tag_stack << tag
-        @text_stack << ""
+        @text_stack << +""
       end
 
       def pop_stacks

data/test/japanese-date-parser-test.rb

@@ -0,0 +1,27 @@
+class JapaneseDateParserTest < Test::Unit::TestCase
+  def setup
+    @parser = Datasets::JapaneseDateParser.new
+  end
+
+  data("month and day with leading a space in Heisei", ["H10.01.01", "平成10年 1月 1日"])
+  data("month         with leading a space in Heisei", ["H10.01.10", "平成10年 1月10日"])
+  data("          day with leading a space in Heisei", ["H10.10.01", "平成10年10月 1日"])
+  data("           without leading a space in Heisei", ["H10.10.10", "平成10年10月10日"])
+  data("year, month and day with leading a space in Reiwa", ["R02.01.01", "令和 2年 1月 1日"])
+  data("year, month         with leading a space in Reiwa", ["R02.01.10", "令和 2年 1月10日"])
+  data("year,           day with leading a space in Reiwa", ["R02.10.01", "令和 2年10月 1日"])
+  data("year,            without leading a space in Reiwa", ["R02.10.10", "令和 2年10月10日"])
+  data("boundary within Heisei", ["H31.04.30", "平成31年 4月30日"])
+  data("boundary within Reiwa", ["R01.05.01", "令和元年 5月 1日"])
+  test("#parse") do
+    expected_jisx0301, japanese_date_string = data
+    assert_equal(expected_jisx0301, @parser.parse(japanese_date_string).jisx0301)
+  end
+
+  test("unsupported era initial range") do
+    expected_message = "era must be one of [平成, 令和]: 昭和"
+    assert_raise(Datasets::JapaneseDateParser::UnsupportedEraInitialRange.new(expected_message)) do
+      @parser.parse("昭和元年 1月 1日")
+    end
+  end
+end

data/test/test-adult.rb

@@ -9,49 +9,24 @@ class AdultTest < Test::Unit::TestCase
     end
 
     test("#each") do
-      records = @dataset.each.to_a
-      assert_equal([
-                     32561,
-                     {
-                       :age => 39,
-                       :work_class => "State-gov",
-                       :final_weight => 77516,
-                       :education => "Bachelors",
-                       :n_education_years => 13,
-                       :marital_status => "Never-married",
-                       :occupation => "Adm-clerical",
-                       :relationship => "Not-in-family",
-                       :race => "White",
-                       :sex => "Male",
-                       :capital_gain => 2174,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => "<=50K"
-                     },
-                     {
-                       :age => 52,
-                       :work_class => "Self-emp-inc",
-                       :final_weight => 287927,
-                       :education => "HS-grad",
-                       :n_education_years => 9,
-                       :marital_status => "Married-civ-spouse",
-                       :occupation => "Exec-managerial",
-                       :relationship => "Wife",
-                       :race => "White",
-                       :sex => "Female",
-                       :capital_gain => 15024,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => ">50K"
-                     }
-                   ],
-                   [
-                     records.size,
-                     records[0].to_h,
-                     records[-1].to_h
-                   ])
+      assert_equal({
+                     :age => 39,
+                     :work_class => "State-gov",
+                     :final_weight => 77516,
+                     :education => "Bachelors",
+                     :n_education_years => 13,
+                     :marital_status => "Never-married",
+                     :occupation => "Adm-clerical",
+                     :relationship => "Not-in-family",
+                     :race => "White",
+                     :sex => "Male",
+                     :capital_gain => 2174,
+                     :capital_loss => 0,
+                     :hours_per_week => 40,
+                     :native_country => "United-States",
+                     :label => "<=50K"
+                   },
+                   @dataset.each.next.to_h)
     end
   end
 
@@ -65,49 +40,24 @@ class AdultTest < Test::Unit::TestCase
     end
 
     test("#each") do
-      records = @dataset.each.to_a
-      assert_equal([
-                     16281,
-                     {
-                       :age => 25,
-                       :work_class => "Private",
-                       :final_weight => 226802,
-                       :education => "11th",
-                       :n_education_years => 7,
-                       :marital_status => "Never-married",
-                       :occupation => "Machine-op-inspct",
-                       :relationship => "Own-child",
-                       :race => "Black",
-                       :sex => "Male",
-                       :capital_gain => 0,
-                       :capital_loss => 0,
-                       :hours_per_week => 40,
-                       :native_country => "United-States",
-                       :label => "<=50K."
-                     },
-                     {
-                       :age => 35,
-                       :work_class => "Self-emp-inc",
-                       :final_weight => 182148,
-                       :education => "Bachelors",
-                       :n_education_years => 13,
-                       :marital_status => "Married-civ-spouse",
-                       :occupation => "Exec-managerial",
-                       :relationship => "Husband",
-                       :race => "White",
-                       :sex => "Male",
-                       :capital_gain => 0,
-                       :capital_loss => 0,
-                       :hours_per_week => 60,
-                       :native_country => "United-States",
-                       :label => ">50K."
-                     }
-                   ],
-                   [
-                     records.size,
-                     records[0].to_h,
-                     records[-1].to_h
-                   ])
+      assert_equal({
+                     :age => 25,
+                     :work_class => "Private",
+                     :final_weight => 226802,
+                     :education => "11th",
+                     :n_education_years => 7,
+                     :marital_status => "Never-married",
+                     :occupation => "Machine-op-inspct",
+                     :relationship => "Own-child",
+                     :race => "Black",
+                     :sex => "Male",
+                     :capital_gain => 0,
+                     :capital_loss => 0,
+                     :hours_per_week => 40,
+                     :native_country => "United-States",
+                     :label => "<=50K.",
+                   },
+                   @dataset.each.next.to_h)
     end
   end
 

data/test/test-aozora-bunko.rb

@@ -114,13 +114,13 @@ class AozoraBunkoTest < Test::Unit::TestCase
           book = Datasets::AozoraBunko::Book.new
           book.cache_path = @cache_path
 
-          book.title_id = '000750'
-          book.person_id = '000146'
-          book.html_file_url = 'http://www.lcv.ne.jp/~ibs52086/fire/'
+          book.title_id = '061551'
+          book.person_id = '002225'
+          book.html_file_url = 'http://minken.party/2019/03/09/yakunin/'
           book.html_file_character_encoding = 'UTF-8'
 
-          assert_equal('<title>種田山頭火句集 | 『草木塔抄』他　FIRE ON THE MOUNTAIN</title>',
-                       book.html.split("\n")[7])
+          assert_equal('<title>自由民権現代研究会</title>',
+                       book.html[/<title>.*?<\/title>/])
         end
       end
 

data/test/test-california-housing.rb

@@ -8,37 +8,18 @@ class CaliforniaHousingTest < Test::Unit::TestCase
   end
 
   test("#each") do
-    records = @dataset.each.to_a
-    assert_equal([
-                   20640,
-                   {
-                     median_house_value: 452600.000000,
-                     median_income: 8.325200,
-                     housing_median_age: 41.000000,
-                     total_rooms: 880.000000,
-                     total_bedrooms: 129.000000,
-                     population: 322.000000,
-                     households: 126.000000,
-                     latitude: 37.880000,
-                     longitude: -122.230000
-                   },
-                   {
-                     median_house_value: 89400.000000,
-                     median_income: 2.388600,
-                     housing_median_age: 16.000000,
-                     total_rooms: 2785.000000,
-                     total_bedrooms: 616.000000,
-                     population: 1387.000000,
-                     households: 530.000000,
-                     latitude: 39.370000,
-                     longitude: -121.240000
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h
-                 ])
+    assert_equal({
+                   median_house_value: 452600.000000,
+                   median_income: 8.325200,
+                   housing_median_age: 41.000000,
+                   total_rooms: 880.000000,
+                   total_bedrooms: 129.000000,
+                   population: 322.000000,
+                   households: 126.000000,
+                   latitude: 37.880000,
+                   longitude: -122.230000,
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do

data/test/test-cldr-plurals.rb

@@ -14,7 +14,7 @@ class CLDRPluralsTest < Test::Unit::TestCase
   test("#each") do
     locales = @dataset.each.to_a
     assert_equal([
-                   219,
+                   222,
                    locale("bm",
                           [
                             rule("other",

data/test/test-diamonds.rb

@@ -8,39 +8,19 @@ class DiamondsTest < Test::Unit::TestCase
   end
 
   test("#each") do
-    records = @dataset.each.to_a
-    assert_equal([
-                   53940,
-                   {
-                     carat: 0.23,
-                     clarity: "SI2",
-                     color: "E",
-                     cut: "Ideal",
-                     depth: 61.5,
-                     price: 326,
-                     table: 55.0,
-                     x: 3.95,
-                     y: 3.98,
-                     z: 2.43,
-                   },
-                   {
-                     carat: 0.75,
-                     clarity: "SI2",
-                     color: "D",
-                     cut: "Ideal",
-                     depth: 62.2,
-                     price: 2757,
-                     table: 55.0,
-                     x: 5.83,
-                     y: 5.87,
-                     z: 3.64,
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h
-                 ])
+    assert_equal({
+                   carat: 0.23,
+                   clarity: "SI2",
+                   color: "E",
+                   cut: "Ideal",
+                   depth: 61.5,
+                   price: 326,
+                   table: 55.0,
+                   x: 3.95,
+                   y: 3.98,
+                   z: 2.43,
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do

data/test/test-downloader.rb

@@ -17,7 +17,7 @@ class DownloaderTest < Test::Unit::TestCase
       output_path = @tmp_dir + "file"
       downloader = Datasets::Downloader.new(first_url)
 
-      downloader.define_singleton_method(:start_http) do |url, headers|
+      downloader.define_singleton_method(:start_http) do |url, fallback_urls, headers|
         raise Datasets::Downloader::TooManyRedirects, "too many redirections: #{last_url}"
       end
 

data/test/test-geolonia.rb

@@ -4,47 +4,23 @@ class GeoloniaTest < Test::Unit::TestCase
   end
 
   test('#each') do
-    records = @dataset.each.to_a
-    assert_equal([
-                   277616,
-                   {
-                     :prefecture_code => "01",
-                     :prefecture_name => "北海道",
-                     :prefecture_kana => "ホッカイドウ",
-                     :prefecture_romaji => "HOKKAIDO",
-                     :municipality_code => "01101",
-                     :municipality_name => "札幌市中央区",
-                     :municipality_kana => "サッポロシチュウオウク",
-                     :municipality_romaji => "SAPPORO SHI CHUO KU",
-                     :street_name => "旭ケ丘一丁目",
-                     :street_kana => "アサヒガオカ 1",
-                     :street_romaji => "ASAHIGAOKA 1",
-                     :alias => nil,
-                     :latitude => "43.04223",
-                     :longitude => "141.319722"
-                   },
-                   {
-                     :prefecture_code => "47",
-                     :prefecture_name => "沖縄県",
-                     :prefecture_kana => "オキナワケン",
-                     :prefecture_romaji => "OKINAWA KEN",
-                     :municipality_code => "47382",
-                     :municipality_name => "八重山郡与那国町",
-                     :municipality_kana => "ヤエヤマグンヨナグニチョウ",
-                     :municipality_romaji => "YAEYAMA GUN YONAGUNI CHO",
-                     :street_name => "字与那国",
-                     :street_kana => nil,
-                     :street_romaji => nil,
-                     :alias => nil,
-                     :latitude => "24.455925",
-                     :longitude => "122.987678",
-                   },
-                 ],
-                 [
-                   records.size,
-                   records[0].to_h,
-                   records[-1].to_h,
-                 ])
+    assert_equal({
+                   :prefecture_code => "01",
+                   :prefecture_name => "北海道",
+                   :prefecture_kana => "ホッカイドウ",
+                   :prefecture_romaji => "HOKKAIDO",
+                   :municipality_code => "01101",
+                   :municipality_name => "札幌市中央区",
+                   :municipality_kana => "サッポロシチュウオウク",
+                   :municipality_romaji => "SAPPORO SHI CHUO KU",
+                   :street_name => "旭ケ丘一丁目",
+                   :street_kana => "アサヒガオカ 1",
+                   :street_romaji => "ASAHIGAOKA 1",
+                   :alias => nil,
+                   :latitude => "43.04223",
+                   :longitude => "141.319722",
+                 },
+                 @dataset.each.next.to_h)
   end
 
   sub_test_case("#metadata") do

data/test/test-house-of-councillor.rb

@@ -0,0 +1,223 @@
+class HouseOfCouncillorTest < Test::Unit::TestCase
+  test("invalid") do
+    message = ":type must be one of [:bill, :in_house_group, :member, :question]: :invalid"
+    assert_raise(ArgumentError.new(message)) do
+      Datasets::HouseOfCouncillor.new(type: :invalid)
+    end
+  end
+
+  sub_test_case(":bill") do
+    def setup
+      @dataset = Datasets::HouseOfCouncillor.new
+    end
+
+    def record(*args)
+      Datasets::HouseOfCouncillor::Bill.new(*args)
+    end
+
+    test("#each") do
+      assert_equal(record(153,
+                          "法律案（内閣提出）",
+                          153,
+                          1,
+                          "司法制度改革推進法案",
+                          "https://www.sangiin.go.jp/japanese/joho1/kousei/gian/153/meisai/m15303153001.htm",
+                          "https://www.sangiin.go.jp/japanese/joho1/kousei/gian/153/pdf/5315310.pdf",
+                          nil,
+                          Date.parse("2001-09-28"),
+                          Date.parse("2001-10-30"),
+                          nil,
+                          "衆先議",
+                          nil,
+                          nil,
+                          nil,
+                          nil,
+                          Date.parse("2001-10-31"),
+                          "法務委員会",
+                          Date.parse("2001-11-08"),
+                          "可決",
+                          Date.parse("2001-11-09"),
+                          "可決",
+                          nil,
+                          "多数",
+                          "押しボタン",
+                          "https://www.sangiin.go.jp/japanese/joho1/kousei/vote/153/153-1109-v005.htm",
+                          Date.parse("2001-10-18"),
+                          "法務委員会",
+                          Date.parse("2001-10-26"),
+                          "可決",
+                          Date.parse("2001-10-30"),
+                          "可決",
+                          nil,
+                          "多数",
+                          "起立",
+                          Date.parse("2001-11-16"),
+                          119,
+                          nil,
+                          nil),
+                   @dataset.each.next)
+    end
+  end
+
+  sub_test_case(":in_house_group") do
+    def setup
+      @dataset = Datasets::HouseOfCouncillor.new(type: :in_house_group)
+    end
+
+    def record(*args)
+      Datasets::HouseOfCouncillor::InHouseGroup.new(*args)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     10,
+                     record(Date.parse("2024-01-26"),
+                            "自由民主党",
+                            "自民",
+                            Date.parse("2024-08-30"),
+                            114,
+                            23,
+                            Date.parse("2025-07-28"),
+                            19,
+                            5,
+                            33,
+                            6,
+                            52,
+                            11,
+                            Date.parse("2028-07-25"),
+                            18,
+                            5,
+                            44,
+                            7,
+                            62,
+                            12),
+                     record(Date.parse("2024-01-26"),
+                            "各派に属しない議員",
+                            "無所属",
+                            Date.parse("2024-08-30"),
+                            12,
+                            4,
+                            Date.parse("2025-07-28"),
+                            1,
+                            0,
+                            7,
+                            2,
+                            8,
+                            2,
+                            Date.parse("2028-07-25"),
+                            1,
+                            0,
+                            3,
+                            2,
+                            4,
+                            2),
+                   ],
+                   [
+                     records.size,
+                     records.first,
+                     records.last,
+                   ])
+    end
+  end
+
+  sub_test_case(":member") do
+    def setup
+      @dataset = Datasets::HouseOfCouncillor.new(type: :member)
+    end
+
+    def record(*args)
+      Datasets::HouseOfCouncillor::Member.new(*args)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     245,
+                     record("足立　敏之",
+                            nil,
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/giin/profile/7016001.htm",
+                            "あだち　としゆき",
+                            "自民",
+                            "比例",
+                            Date.parse("2028-07-25"),
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/giin/photo/g7016001.jpg",
+                            [2016, 2022],
+                            2,
+                            "財政金融委員会（長）",
+                            Date.parse("2024-08-30"),
+                            "昭和29年5月20日兵庫県西宮市生まれ。（本籍地・京都府福知山市）昭和48年和歌山県立桐蔭高等学校卒業、昭和52年京都大学工学部土木工学科卒業、昭和54年京都大学大学院工学研究科修士課程修了、同年建設省入省後、兵庫県庁、東北及び関東地方整備局、河川局河川計画課河川事業調整官、内閣官房（安全保障・危機管理担当）等を経て、平成15年近畿地方整備局企画部長、平成18年河川局河川計画課長、平成21年四国地方整備局長、平成23年中部地方整備局長、平成24年水管理・国土保全局長、平成25年技監、平成26年国土交通省を退職。平成28年第24回参議院議員通常選挙で初当選○参議院予算委員会理事、災害対策特別委員会理事○著書「激甚化する水害」「いいね！建設産業本当の魅力」（日経BP社）",
+                            Date.parse("2022-11-30")),
+                     record("渡辺　猛之",
+                            nil,
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/giin/profile/7010055.htm",
+                            "わたなべ　たけゆき",
+                            "自民",
+                            "岐阜",
+                            Date.parse("2028-07-25"),
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/giin/photo/g7010055.jpg",
+                            [2010, 2016, 2022],
+                            3,
+                            "経済産業委員会、議院運営委員会（理）",
+                            Date.parse("2024-08-30"),
+                            "昭和43年4月18日生、岐阜県加茂郡八百津町出身。岐阜県立加茂高等学校、名古屋大学経済学部卒業。平成4年、財団法人松下政経塾入塾（第13期生）。平成7年、同塾卒業後、26歳で岐阜県議会議員に初当選。以後通算4期当選。在任中は、自民党岐阜県連副幹事長、岐阜県商工会青年部連合会会長、岐阜県商工政治連盟会長、県監査委員、県政自民クラブ幹事長を歴任。平成22年7月、参議院議員初当選○農林水産委員長、政治倫理の確立及び選挙制度に関する特別委員長、参議院自民党筆頭副幹事長、国土交通副大臣兼内閣府副大臣兼復興副大臣を歴任○現在議院運営委員会筆頭理事。環境委員",
+                            Date.parse("2022-11-30")),
+                   ],
+                   [
+                     records.size,
+                     records.first,
+                     records.last,
+                   ])
+    end
+  end
+
+  sub_test_case(":question") do
+    def setup
+      @dataset = Datasets::HouseOfCouncillor.new(type: :question)
+    end
+
+    def record(*args)
+      Datasets::HouseOfCouncillor::Question.new(*args)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     7833,
+                     record(1,
+                            1,
+                            "食生活安定に関する質問主意書",
+                            "市来　乙彦",
+                            1,
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/001/syuh/s001001.htm",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/001/touh/t001001.htm",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/001/syup/s001001.pdf",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/001/toup/t001001.pdf",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/001/meisai/m001001.htm",
+                            Date.parse("1947-06-06"),
+                            Date.parse("1947-06-23"),
+                            Date.parse("1947-06-28"),
+                            nil),
+                     record(213,
+                            239,
+                            "地方自治法第二百五十二条の二十六の五に定める各大臣の「生命等の保護の措置に関する指示」と地方自治の本旨等との関係に関する質問主意書",
+                            "小西　洋之",
+                            1,
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/syuh/s213239.htm",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/touh/t213239.htm",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/syup/s213239.pdf",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/toup/t213239.pdf",
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/meisai/m213239.htm",
+                            Date.parse("2024-06-21"),
+                            Date.parse("2024-06-21"),
+                            Date.parse("2024-07-02"),
+                            "6月25日内閣から通知書受領（7月2日まで答弁延期）"),
+                   ],
+                   [
+                     records.size,
+                     records.first,
+                     records.last,
+                   ])
+    end
+  end
+end