red-datasets 0.1.4 → 0.1.6

data/test/test-pmjt-dataset-list.rb

@@ -0,0 +1,50 @@
+class PMJTDatasetListTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets:: PMJTDatasetList.new
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+
+    record_first = Datasets::PMJTDatasetList::Record.new
+    record_first.unit = '冊'
+    record_first.open_data_category = '総記'
+    record_first.tag = nil
+    record_first.release_time = 'H31.1'
+    record_first.n_volumes = '2'
+    record_first.type = '刊'
+    record_first.publication_year = '元禄９'
+    record_first.original_request_code = '９９－３７－１～２'
+    record_first.id = '200003090'
+    record_first.title = '人倫重宝記'
+    record_first.text = nil
+    record_first.bibliographical_introduction = nil
+    record_first.year = nil
+
+    record_last = Datasets::PMJTDatasetList::Record.new
+    record_last.unit = '冊'
+    record_last.open_data_category = '総記'
+    record_last.tag = nil
+    record_last.release_time = 'H27.11'
+    record_last.n_volumes = '1'
+    record_last.type = '刊'
+    record_last.publication_year = '慶応2'
+    record_last.original_request_code = '４９－１７３'
+    record_last.id = '200021837'
+    record_last.title = '洋学便覧'
+    record_last.text = nil
+    record_last.bibliographical_introduction = '○'
+    record_last.year = '1866'
+
+    assert_equal([
+                    3126,
+                    record_first,
+                    record_last
+                  ],
+                  [
+                    records.size,
+                    records[1],
+                    records[-1]
+                  ])
+  end
+end

data/test/test-quora-duplicate-question-pair.rb

@@ -0,0 +1,33 @@
+class QuoraDuplicateQuestionPairTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::QuoraDuplicateQuestionPair.new
+  end
+
+  def record(*args)
+    Datasets::QuoraDuplicateQuestionPair::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   404290,
+                   record(0,
+                          1,
+                          2,
+                          "What is the step by step guide to invest in share market in india?",
+                          "What is the step by step guide to invest in share market?",
+                          false),
+                   record(404289,
+                          537932,
+                          537933,
+                          "What is like to have sex with cousin?",
+                          "What is it like to have sex with your cousin?",
+                          false),
+                 ],
+                 [
+                   records.size,
+                   records.first,
+                   records.last,
+                 ])
+  end
+end

data/test/test-rdataset.rb

@@ -0,0 +1,246 @@
+class RdatasetTest < Test::Unit::TestCase
+  sub_test_case("RdatasetList") do
+    def setup
+      @dataset = Datasets::RdatasetList.new
+    end
+
+    sub_test_case("#each") do
+      test("with package_name") do
+        records = @dataset.filter(package: "datasets").to_a
+        assert_equal([
+                       102,
+                       {
+                         package: "datasets",
+                         dataset: "ability.cov",
+                         title: "Ability and Intelligence Tests",
+                         rows: 6,
+                         cols: 8,
+                         n_binary: 0,
+                         n_character: 0,
+                         n_factor: 0,
+                         n_logical: 0,
+                         n_numeric: 8,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/ability.cov.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/ability.cov.html"
+                       },
+                       {
+                         package: "datasets",
+                         dataset: "WWWusage",
+                         title: "Internet Usage per Minute",
+                         rows: 100,
+                         cols: 2,
+                         n_binary: 0,
+                         n_character: 0,
+                         n_factor: 0,
+                         n_logical: 0,
+                         n_numeric: 2,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/WWWusage.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/WWWusage.html"
+                       }
+                     ],
+                     [
+                       records.size,
+                       records[0].to_h,
+                       records[-1].to_h
+                     ])
+      end
+
+      test("without package_name") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       2142,
+                       {
+                         package: "AER",
+                         dataset: "Affairs",
+                         title: "Fair's Extramarital Affairs Data",
+                         rows: 601,
+                         cols: 9,
+                         n_binary: 2,
+                         n_character: 0,
+                         n_factor: 2,
+                         n_logical: 0,
+                         n_numeric: 7,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/AER/Affairs.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/AER/Affairs.html"
+                       },
+                       {
+                         package: "wooldridge",
+                         dataset: "wine",
+                         title: "wine",
+                         rows: 21,
+                         cols: 5,
+                         n_binary: 0,
+                         n_character: 1,
+                         n_factor: 0,
+                         n_logical: 0,
+                         n_numeric: 4,
+                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/wooldridge/wine.csv",
+                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/wooldridge/wine.html"
+                       },
+                     ],
+                     [
+                       records.size,
+                       records[0].to_h,
+                       records[-1].to_h
+                     ])
+      end
+    end
+  end
+
+  sub_test_case("Rdataset") do
+    test('invalid package name') do
+      assert_raise(ArgumentError) do
+        Datasets::Rdataset.new('invalid package name', 'AirPassengers')
+      end
+    end
+
+    sub_test_case("datasets") do
+      test("invalid dataset name") do
+        assert_raise(ArgumentError) do
+          Datasets::Rdataset.new("datasets", "invalid datasets name")
+        end
+      end
+
+      sub_test_case("AirPassengers") do
+        def setup
+          @dataset = Datasets::Rdataset.new("datasets", "AirPassengers")
+        end
+
+        test("#each") do
+          records = @dataset.each.to_a
+          assert_equal([
+                         144,
+                         { time: 1949,             value: 112 },
+                         { time: 1960.91666666667, value: 432 },
+                       ],
+                       [
+                         records.size,
+                         records[0],
+                         records[-1]
+                       ])
+        end
+
+        test("#metadata.id") do
+          assert_equal("rdataset-datasets-AirPassengers", @dataset.metadata.id)
+        end
+
+        test("#metadata.description") do
+          description = @dataset.metadata.description
+          assert do
+            description.include?("Monthly Airline Passenger Numbers 1949-1960")
+          end
+        end
+      end
+
+      sub_test_case("airquality") do
+        def setup
+          @dataset = Datasets::Rdataset.new("datasets", "airquality")
+        end
+
+        test("#each") do
+          records = @dataset.each.to_a
+          assert_equal([
+                         153,
+                         { Ozone: nil, "Solar.R": nil, Wind: 14.3, Temp: 56, Month: 5, Day: 5 },
+                         { Ozone: 20, "Solar.R": 223, Wind: 11.5, Temp: 68, Month: 9, Day: 30 },
+                       ],
+                       [
+                         records.size,
+                         records[4],
+                         records[-1]
+                       ])
+        end
+      end
+
+      sub_test_case('attenu') do
+        def setup
+          @dataset = Datasets::Rdataset.new('datasets', 'attenu')
+        end
+
+        test('#each') do
+          records = @dataset.each.to_a
+          assert_equal([
+                         182,
+                         { event: 1, mag: 7, station: "117", dist: 12, accel: 0.359 },
+                         { event: 16, mag: 5.1, station: nil, dist: 7.6, accel: 0.28 },
+                         { event: 23, mag: 5.3, station: "c168", dist: 25.3, accel: 0.23 },
+                         { event: 23, mag: 5.3, station: "5072", dist: 53.1, accel: 0.022 }
+                       ],
+                       [
+                         records.size,
+                         records[0],
+                         records[78],
+                         records[169],
+                         records[-1]
+                       ])
+        end
+      end
+    end
+
+    sub_test_case('drc') do
+      sub_test_case('germination') do
+        def setup
+          @dataset = Datasets::Rdataset.new('drc', 'germination')
+        end
+
+        test('#each') do
+          records = @dataset.each.to_a
+          assert_equal([
+                         192,
+                         { temp: 10, species: 'wheat', start: 0, end: 1.0, germinated: 0 },
+                         { temp: 40, species: 'rice', start: 18, end: Float::INFINITY, germinated: 12 }
+                       ],
+                       [
+                         records.size,
+                         records[0],
+                         records[-1]
+                       ])
+        end
+      end
+    end
+
+    sub_test_case('validate') do
+      sub_test_case('nace_rev2') do
+        def setup
+          @dataset = Datasets::Rdataset.new('validate', 'nace_rev2')
+        end
+
+        test('#each') do
+          records = @dataset.each.to_a
+          assert_equal([
+                        996,
+                        {
+                          Order: 398_481,
+                          Level: 1,
+                          Code: 'A',
+                          Parent: '',
+                          Description: 'AGRICULTURE, FORESTRY AND FISHING',
+                          This_item_includes: 'This section includes the exploitation of vegetal and animal natural resources, comprising the activities of growing of crops, raising and breeding of animals, harvesting of timber and other plants, animals or animal products from a farm or their natural habitats.',
+                          This_item_also_includes: '',
+                          Rulings: '',
+                          This_item_excludes: '',
+                          "Reference_to_ISIC_Rev._4": 'A'
+                        },
+                        {
+                          Order: 399_476,
+                          Level: 4,
+                          Code: '99.00',
+                          Parent: '99.0',
+                          Description: 'Activities of extraterritorial organisations and bodies',
+                          This_item_includes: "This class includes:\n- activities of international organisations such as the United Nations and the specialised agencies of the United Nations system, regional bodies etc., the International Monetary Fund, the World Bank, the World Customs Organisation, the Organisation for Economic Co-operation and Development, the organisation of Petroleum Exporting Countries, the European Communities, the European Free Trade Association etc.",
+                          This_item_also_includes: "This class also includes:\n- activities of diplomatic and consular missions when being determined by the country of their location rather than by the country they represent",
+                          Rulings: '',
+                          This_item_excludes: '',
+                          "Reference_to_ISIC_Rev._4": '9900'
+                        }
+                      ],
+                      [
+                        records.size,
+                        records[0],
+                        records[-1]
+                      ])
+        end
+      end
+    end
+  end
+end

data/test/{test-seaborn-data.rb → test-seaborn.rb}

@@ -1,7 +1,42 @@
-class SeabornDataTest < Test::Unit::TestCase
+class SeabornTest < Test::Unit::TestCase
+  sub_test_case("list") do
+    def setup
+      @dataset = Datasets::SeabornList.new
+    end
+
+    def test_each
+      records = @dataset.each.to_a
+      assert_equal([
+                     {dataset: "anagrams"},
+                     {dataset: "anscombe"},
+                     {dataset: "attention"},
+                     {dataset: "brain_networks"},
+                     {dataset: "car_crashes"},
+                     {dataset: "diamonds"},
+                     {dataset: "dots"},
+                     {dataset: "dowjones"},
+                     {dataset: "exercise"},
+                     {dataset: "flights"},
+                     {dataset: "fmri"},
+                     {dataset: "geyser"},
+                     {dataset: "glue"},
+                     {dataset: "healthexp"},
+                     {dataset: "iris"},
+                     {dataset: "mpg"},
+                     {dataset: "penguins"},
+                     {dataset: "planets"},
+                     {dataset: "seaice"},
+                     {dataset: "taxis"},
+                     {dataset: "tips"},
+                     {dataset: "titanic"},
+                   ],
+                   records)
+    end
+  end
+
   sub_test_case("fmri") do
     def setup
-      @dataset = Datasets::SeabornData.new("fmri")
+      @dataset = Datasets::Seaborn.new("fmri")
     end
 
     def test_each
@@ -33,7 +68,7 @@ class SeabornDataTest < Test::Unit::TestCase
 
   sub_test_case("flights") do
     def setup
-      @dataset = Datasets::SeabornData.new("flights")
+      @dataset = Datasets::Seaborn.new("flights")
     end
 
     def test_each
@@ -61,7 +96,7 @@ class SeabornDataTest < Test::Unit::TestCase
 
   sub_test_case("penguins") do
     def setup
-      @dataset = Datasets::SeabornData.new("penguins")
+      @dataset = Datasets::Seaborn.new("penguins")
     end
 
     def test_each
@@ -94,4 +129,36 @@ class SeabornDataTest < Test::Unit::TestCase
                    ])
     end
   end
+
+  sub_test_case("attention") do
+    def setup
+      @dataset = Datasets::Seaborn.new("attention")
+    end
+
+    def test_each
+      records = @dataset.to_a
+      assert_equal([
+                     60,
+                     {
+                       index: 1,
+                       subject: 2,
+                       attention: "divided",
+                       solutions: 1,
+                       score: 3.0
+                     },
+                     {
+                       index: 59,
+                       subject: 20,
+                       attention: "focused",
+                       solutions: 3,
+                       score: 5.0
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[1],
+                     records[-1]
+                   ])
+    end
+  end
 end

data/test/test-sudachi-synonym-dictionary.rb

@@ -6,7 +6,7 @@ class SudachiSynonymDictionaryTest < Test::Unit::TestCase
   test('#each') do
     records = @dataset.each.to_a
     assert_equal([
-                   61335,
+                   65206,
                    {
                      group_id: "000001",
                      is_noun: true,
@@ -19,15 +19,15 @@ class SudachiSynonymDictionaryTest < Test::Unit::TestCase
                      notation: "曖昧",
                    },
                    {
-                     group_id: "023705",
+                     group_id: "024916",
                      is_noun: true,
-                     expansion_type: :always,
+                     expansion_type: :expanded,
                      lexeme_id: 1,
                      form_type: :typical,
                      acronym_type: :alphabet,
                      variant_type: :typical,
-                     categories: ["単位"],
-                     notation: "GB",
+                     categories: ["ビジネス"],
+                     notation: "SCM",
                    },
                  ],
                  [

data/test/test-wikipedia-kyoto-japanese-english.rb

@@ -0,0 +1,178 @@
+class WikipediaKyotoJapaneseEnglishTest < Test::Unit::TestCase
+  sub_test_case("article") do
+    def setup
+      @dataset = Datasets::WikipediaKyotoJapaneseEnglish.new(type: :article)
+    end
+
+    def shorten_text(text)
+      max = 20
+      if text.size <= max
+        text
+      else
+        "#{text[0, max]}..."
+      end
+    end
+
+    def hashify(record)
+      hash = {class: record.class.name.split("::").last}
+      case record
+      when Datasets::WikipediaKyotoJapaneseEnglish::Title
+        hash[:section] = record.section&.id
+        hash[:japanese] = shorten_text(record.japanese)
+        hash[:english] = shorten_text(record.english)
+      when Datasets::WikipediaKyotoJapaneseEnglish::Sentence
+        hash[:id] = record.id
+        hash[:section] = record.section&.id
+        hash[:paragraph] = record.paragraph&.id
+        hash[:japanese] = shorten_text(record.japanese)
+        hash[:english] = shorten_text(record.english)
+      else
+        record.members.each do |member|
+          value = record[member]
+          case value
+          when Array
+            value = value.collect do |v|
+              hashify(v)
+            end
+          when String
+            value = shorten_text(value)
+          when Struct
+            value = hasify(value)
+          end
+          hash[member] = value
+        end
+      end
+      hash
+    end
+
+    test("#each") do
+      first_record = @dataset.each.first
+      assert_equal({
+                     class: "Article",
+                     copyright: "copyright (c) 2010 前...",
+                     sections: [],
+                     source: "jawiki-20080607-page...",
+                     contents: [
+                       {
+                         class: "Title",
+                         section: nil,
+                         english: "Genkitsu SANYO",
+                         japanese: "三要元佶",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "1",
+                         section: nil,
+                         paragraph: "1",
+                         english: "Genkitsu SANYO (1548...",
+                         japanese: "三要元佶（さんよう げんきつ, 天文 (...",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "2",
+                         section: nil,
+                         paragraph: "2",
+                         english: "He was originally fr...",
+                         japanese: "肥前国（佐賀県）の出身。",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "3",
+                         section: nil,
+                         paragraph: "2",
+                         english: "His Go (pen name) wa...",
+                         japanese: "号は閑室。",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "4",
+                         section: nil,
+                         paragraph: "2",
+                         english: "He was called Kiccho...",
+                         japanese: "佶長老、閑室和尚と呼ばれた。",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "5",
+                         section: nil,
+                         paragraph: "3",
+                         english: "He went up to the ca...",
+                         japanese: "幼少時に都に上り、岩倉の円通寺 (京都市...",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "6",
+                         section: nil,
+                         paragraph: "4",
+                         english: "After assuming the p...",
+                         japanese: "足利学校の長となるが、関ヶ原の戦いの折に...",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "7",
+                         section: nil,
+                         paragraph: "5",
+                         english: "He assumed the posit...",
+                         japanese: "金地院崇伝と寺社奉行の任に当たり、西笑承...",
+                       },
+                       {
+                         class: "Sentence",
+                         id: "8",
+                         section: nil,
+                         paragraph: "6",
+                         english: "Later, he was invite...",
+                         japanese: "家康によって、伏見区の学校に招かれ、円光...",
+                       },
+                     ],
+                   },
+                   hashify(first_record))
+    end
+  end
+
+  sub_test_case("lexicon") do
+    def setup
+      @dataset = Datasets::WikipediaKyotoJapaneseEnglish.new(type: :lexicon)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     51982,
+                     {
+                       :japanese => "102世吉田日厚貫首",
+                       :english => "the 102nd head priest, Nikko TOSHIDA"
+                     },
+                     {
+                       :japanese => "龗神社",
+                       :english => "Okami-jinja Shrine"
+                     },
+                    ],
+                    [
+                      records.size,
+                      records[0].to_h,
+                      records[-1].to_h,
+                    ])
+    end
+  end
+
+  test("invalid") do
+    message = "Please set type :article or :lexicon: :invalid"
+    assert_raise(ArgumentError.new(message)) do
+      Datasets::WikipediaKyotoJapaneseEnglish.new(type: :invalid)
+    end
+  end
+
+  test("description") do
+    dataset = Datasets::WikipediaKyotoJapaneseEnglish.new
+    description = dataset.metadata.description
+    assert_equal(<<-DESCRIPTION, description)
+"The Japanese-English Bilingual Corpus of Wikipedia's Kyoto Articles"
+aims mainly at supporting research and development relevant to
+high-performance multilingual machine translation, information
+extraction, and other language processing technologies. The National
+Institute of Information and Communications Technology (NICT) has
+created this corpus by manually translating Japanese Wikipedia
+articles (related to Kyoto) into English.
+    DESCRIPTION
+  end
+end