red-datasets 0.1.4 → 0.1.5

data/lib/datasets/zip-extractor.rb

@@ -0,0 +1,36 @@
+require 'zip'
+
+module Datasets
+  class ZipExtractor
+    def initialize(path)
+      @path = path
+    end
+
+    def extract_first_file
+      Zip::File.open(@path) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+
+          entry.get_input_stream do |input|
+            return yield(input)
+          end
+        end
+      end
+      nil
+    end
+
+    def extract_file(file_path)
+      Zip::File.open(@path) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+          next unless entry.name == file_path
+
+          entry.get_input_stream do |input|
+            return yield(input)
+          end
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/datasets.rb

@@ -1,22 +1,34 @@
 require_relative "datasets/version"
 
 require_relative "datasets/adult"
+require_relative "datasets/afinn"
+require_relative "datasets/aozora-bunko"
+require_relative "datasets/california-housing"
 require_relative "datasets/cifar"
 require_relative "datasets/cldr-plurals"
 require_relative "datasets/communities"
+require_relative "datasets/diamonds"
 require_relative "datasets/e-stat-japan"
 require_relative "datasets/fashion-mnist"
+require_relative "datasets/fuel-economy"
+require_relative "datasets/geolonia"
 require_relative "datasets/hepatitis"
 require_relative "datasets/iris"
+require_relative "datasets/ita-corpus"
+require_relative "datasets/kuzushiji-mnist"
 require_relative "datasets/libsvm"
 require_relative "datasets/libsvm-dataset-list"
+require_relative "datasets/livedoor-news"
 require_relative "datasets/mnist"
 require_relative "datasets/mushroom"
 require_relative "datasets/penguins"
 require_relative "datasets/penn-treebank"
+require_relative "datasets/pmjt-dataset-list"
 require_relative "datasets/postal-code-japan"
-require_relative "datasets/rdatasets"
-require_relative "datasets/seaborn-data"
+require_relative "datasets/quora-duplicate-question-pair"
+require_relative "datasets/rdataset"
+require_relative "datasets/seaborn"
 require_relative "datasets/sudachi-synonym-dictionary"
 require_relative "datasets/wikipedia"
+require_relative "datasets/wikipedia-kyoto-japanese-english"
 require_relative "datasets/wine"

data/red-datasets.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("doc/text/*")
   spec.test_files += Dir.glob("test/**/*")
 
-  spec.add_runtime_dependency("csv", ">= 3.0.5")
+  spec.add_runtime_dependency("csv", ">= 3.2.4")
   spec.add_runtime_dependency("rexml")
   spec.add_runtime_dependency("rubyzip")
 

data/test/helper.rb

@@ -1,6 +1,7 @@
 require "fileutils"
 require "pathname"
 require "time"
+require "tmpdir"
 
 require "datasets"
 
@@ -18,4 +19,24 @@ module Helper
       FileUtils.rm_rf(@tmp_dir)
     end
   end
+
+  module PathRestorable
+    def restore_path(path)
+      unless path.exist?
+        return yield
+      end
+
+      Dir.mktmpdir do |dir|
+        FileUtils.cp_r(path, dir, preserve: true)
+        begin
+          yield
+        ensure
+          FileUtils.rmtree(path, secure: true) if path.exist?
+          FileUtils.cp_r(Pathname(dir) + path.basename,
+                         path,
+                         preserve: true)
+        end
+      end
+    end
+  end
 end

data/test/test-afinn.rb

@@ -0,0 +1,60 @@
+class AFINNTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::AFINN.new
+  end
+
+  test('#each') do
+    records = @dataset.each.to_a
+    assert_equal([
+                   2477,
+                   {
+                     :valence => -2,
+                     :word => "abandon"
+                   },
+                   {
+                     :valence => 2,
+                     :word => "zealous"
+                   },
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h,
+                 ])
+  end
+
+  sub_test_case('#metadata') do
+    test('#description') do
+      description = @dataset.metadata.description
+      assert_equal(<<-DESCRIPTION.chomp, description)
+AFINN is a list of English words rated for valence with an integer
+between minus five (negative) and plus five (positive). The words have
+been manually labeled by Finn Årup Nielsen in 2009-2011. The file
+is tab-separated. There are two versions:
+
+AFINN-111: Newest version with 2477 words and phrases.
+
+An evaluation of the word list is available in:
+
+Finn Årup Nielsen, "A new ANEW: Evaluation of a word list for
+sentiment analysis in microblogs", http://arxiv.org/abs/1103.2903
+
+The list was used in: 
+
+Lars Kai Hansen, Adam Arvidsson, Finn Årup Nielsen, Elanor Colleoni,
+Michael Etter, "Good Friends, Bad News - Affect and Virality in
+Twitter", The 2011 International Workshop on Social Computing,
+Network, and Services (SocialComNet 2011).
+
+
+This database of words is copyright protected and distributed under
+"Open Database License (ODbL) v1.0"
+http://www.opendatacommons.org/licenses/odbl/1.0/ or a similar
+copyleft license.
+
+See comments on the word list here:
+http://fnielsen.posterous.com/old-anew-a-sentiment-about-sentiment-analysis
+      DESCRIPTION
+    end
+  end
+end

data/test/test-aozora-bunko.rb

@@ -0,0 +1,190 @@
+class AozoraBunkoTest < Test::Unit::TestCase
+  include Helper::PathRestorable
+
+  def setup
+    @dataset = Datasets::AozoraBunko.new
+    @cache_path = @dataset.send(:cache_path)
+  end
+
+  test('#new') do
+    assert_equal({
+                   title_id: '059898',
+                   title: 'ウェストミンスター寺院',
+                   title_reading: 'ウェストミンスターじいん',
+                   title_reading_collation: 'うえすとみんすたあしいん',
+                   subtitle: '',
+                   subtitle_reading: '',
+                   original_title: '',
+                   first_appearance: '',
+                   ndc_code: 'NDC 933',
+                   syllabary_spelling_type: '新字新仮名',
+                   copyrighted: false,
+                   published_date: '2020-04-03',
+                   last_updated_date: '2020-03-28',
+                   detail_url: 'https://www.aozora.gr.jp/cards/001257/card59898.html',
+                   person_id: '001257',
+                   person_family_name: 'アーヴィング',
+                   person_first_name: 'ワシントン',
+                   person_family_name_reading: 'アーヴィング',
+                   person_first_name_reading: 'ワシントン',
+                   person_family_name_reading_collation: 'ああういんく',
+                   person_first_name_reading_collation: 'わしんとん',
+                   person_family_name_romaji: 'Irving',
+                   person_first_name_romaji: 'Washington',
+                   person_type: '著者',
+                   person_birthday: '1783-04-03',
+                   person_date_of_death: '1859-11-28',
+                   person_copyrighted: false,
+                   original_book_name1: 'スケッチ・ブック',
+                   original_book_publisher_name1: '新潮文庫、新潮社',
+                   original_book_first_published_date1: '1957（昭和32）年5月20日',
+                   used_version_for_registration1: '2000（平成12）年2月20日33刷改版',
+                   used_version_for_proofreading1: '2000（平成12）年2月20日33刷改版',
+                   base_of_original_book_name1: '',
+                   base_of_original_book_publisher_name1: '',
+                   base_of_original_book_first_published_date1: '',
+                   original_book_name2: '',
+                   original_book_publisher_name2: '',
+                   original_book_first_published_date2: '',
+                   used_version_for_registration2: '',
+                   used_version_for_proofreading2: '',
+                   base_of_original_book_name2: '',
+                   base_of_original_book_publisher_name2: '',
+                   base_of_original_book_first_published_date2: '',
+                   registered_person_name: 'えにしだ',
+                   proofreader_name: '砂場清隆',
+                   text_file_url: 'https://www.aozora.gr.jp/cards/001257/files/59898_ruby_70679.zip',
+                   last_text_file_updated_date: '2020-03-28',
+                   text_file_character_encoding: 'ShiftJIS',
+                   text_file_character_set: 'JIS X 0208',
+                   text_file_updating_count: '0',
+                   html_file_url: 'https://www.aozora.gr.jp/cards/001257/files/59898_70731.html',
+                   last_html_file_updated_date: '2020-03-28',
+                   html_file_character_encoding: 'ShiftJIS',
+                   html_file_character_set: 'JIS X 0208',
+                   html_file_updating_count: '0'
+
+                 },
+                 @dataset.first.to_h)
+  end
+
+  sub_test_case(:Book) do
+    sub_test_case('#text') do
+      test('readable') do
+        book = Datasets::AozoraBunko::Book.new
+        book.cache_path = @cache_path
+        book.title_id = '059898'
+        book.person_id = '001257'
+        book.text_file_url = 'https://www.aozora.gr.jp/cards/001257/files/59898_ruby_70679.zip'
+        book.text_file_character_encoding = 'ShiftJIS'
+
+        assert_equal([
+                       'ウェストミンスター寺',
+                       "アの皆さんです。\r\n"
+                     ],
+                     [
+                       book.text[0, 10],
+                       book.text[-10, 10]
+                     ])
+      end
+
+      test('not readable') do
+        book = Datasets::AozoraBunko::Book.new
+        book.text_file_url = 'https://mega.nz/file/6tMxgAjZ#PglDDyJL0syRhnULqK0qhTMC7cktsgqwObj5fY_knpE'
+
+        assert_equal(nil, book.text)
+      end
+    end
+
+    sub_test_case('#html') do
+      sub_test_case('readable') do
+        test('encoding is ShiftJIS') do
+          book = Datasets::AozoraBunko::Book.new
+          book.cache_path = @cache_path
+          book.title_id = '059898'
+          book.person_id = '001257'
+          book.html_file_url = 'https://www.aozora.gr.jp/cards/001257/files/59898_70731.html'
+          book.html_file_character_encoding = 'ShiftJIS'
+
+          assert_equal("<title>ワシントン・アーヴィング　Washington Irving 吉田甲子太郎訳 ウェストミンスター寺院</title>",
+                       book.html.split("\n")[8].strip)
+        end
+
+        test('encoding is UTF-8') do
+          book = Datasets::AozoraBunko::Book.new
+          book.cache_path = @cache_path
+
+          book.title_id = '000750'
+          book.person_id = '000146'
+          book.html_file_url = 'http://www.lcv.ne.jp/~ibs52086/fire/'
+          book.html_file_character_encoding = 'UTF-8'
+
+          assert_equal('<title>種田山頭火句集 | 『草木塔抄』他　FIRE ON THE MOUNTAIN</title>',
+                       book.html.split("\n")[7])
+        end
+      end
+
+      test('not readable') do
+        book = Datasets::AozoraBunko::Book.new
+        book.html_file_url = ''
+
+        assert_equal(nil, book.html)
+      end
+    end
+
+    sub_test_case('converting boolean') do
+      test('#person_copyrighted?') do
+        book = @dataset.first
+        assert_equal([
+                       false,
+                       false,
+                       false,
+                     ],
+                     [
+                       book.person_copyrighted?,
+                       book.person_copyrighted,
+                       book.to_h[:person_copyrighted],
+                     ])
+      end
+
+      test('#copyrighted?') do
+        book = @dataset.first
+        assert_equal([
+                       false,
+                       false,
+                       false,
+                     ],
+                     [
+                       book.copyrighted?,
+                       book.copyrighted,
+                       book.to_h[:copyrighted],
+                     ])
+      end
+    end
+
+    test('#clear_cache! removes all cache files') do
+      book = Datasets::AozoraBunko::Book.new
+      book.cache_path = @cache_path
+
+      book.title_id = '059898'
+      book.person_id = '001257'
+      book.text_file_url = 'https://www.aozora.gr.jp/cards/001257/files/59898_ruby_70679.zip'
+      book.text_file_character_encoding = 'ShiftJIS'
+      book.html_file_url = 'https://www.aozora.gr.jp/cards/001257/files/59898_70731.html'
+      book.html_file_character_encoding = 'ShiftJIS'
+
+      book.text
+      book.html
+
+      restore_path(@cache_path.base_dir) do
+        assert_equal(true, @cache_path.base_dir.exist?)
+        assert_equal(true, book.send(:text_file_output_path).exist?)
+        assert_equal(true, book.send(:html_file_output_path).exist?)
+        @dataset.clear_cache!
+        assert_equal(false, book.send(:html_file_output_path).exist?)
+        assert_equal(false, book.send(:text_file_output_path).exist?)
+        assert_equal(false, @cache_path.base_dir.exist?)
+      end
+    end
+  end
+end

data/test/test-california-housing.rb

@@ -0,0 +1,56 @@
+class CaliforniaHousingTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::CaliforniaHousing.new
+  end
+
+  def record(*args)
+    Datasets::CaliforniaHousing::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   20640,
+                   {
+                     median_house_value: 452600.000000,
+                     median_income: 8.325200,
+                     housing_median_age: 41.000000,
+                     total_rooms: 880.000000,
+                     total_bedrooms: 129.000000,
+                     population: 322.000000,
+                     households: 126.000000,
+                     latitude: 37.880000,
+                     longitude: -122.230000
+                   },
+                   {
+                     median_house_value: 89400.000000,
+                     median_income: 2.388600,
+                     housing_median_age: 16.000000,
+                     total_rooms: 2785.000000,
+                     total_bedrooms: 616.000000,
+                     population: 1387.000000,
+                     households: 530.000000,
+                     latitude: 39.370000,
+                     longitude: -121.240000
+                   },
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert_equal(<<-DESCRIPTION, description)
+Housing information from the 1990 census used in
+Pace, R. Kelley and Ronald Barry,
+"Sparse Spatial Autoregressions",
+Statistics and Probability Letters, 33 (1997) 291-297.
+Available from http://lib.stat.cmu.edu/datasets/.
+      DESCRIPTION
+    end
+  end
+end

data/test/test-cldr-plurals.rb

@@ -14,7 +14,7 @@ class CLDRPluralsTest < Test::Unit::TestCase
   test("#each") do
     locales = @dataset.each.to_a
     assert_equal([
-                   218,
+                   219,
                    locale("bm",
                           [
                             rule("other",

data/test/test-dataset.rb

@@ -1,5 +1,7 @@
 class TestDataset < Test::Unit::TestCase
   sub_test_case("#clear_cache!") do
+    include Helper::PathRestorable
+
     def setup
       @dataset = Datasets::Iris.new
       @cache_dir_path = @dataset.send(:cache_dir_path)
@@ -9,18 +11,24 @@ class TestDataset < Test::Unit::TestCase
       @dataset.first # This ensures the dataset downloaded
       existence = {before: @cache_dir_path.join("iris.csv").exist?}
 
-      @dataset.clear_cache!
-      existence[:after] = @cache_dir_path.join("iris.csv").exist?
+      restore_path(@cache_dir_path) do
+        @dataset.clear_cache!
+        existence[:after] = @cache_dir_path.join("iris.csv").exist?
 
-      assert_equal({before: true, after: false},
-                   existence)
+        assert_equal({before: true, after: false},
+                     existence)
+      end
     end
 
     test("when the dataset is not downloaded") do
-      FileUtils.rmtree(@cache_dir_path.to_s, secure: true) if @cache_dir_path.exist?
+      restore_path(@cache_dir_path) do
+        if @cache_dir_path.exist?
+          FileUtils.rmtree(@cache_dir_path.to_s, secure: true)
+        end
 
-      assert_nothing_raised do
-        @dataset.clear_cache!
+        assert_nothing_raised do
+          @dataset.clear_cache!
+        end
       end
     end
   end

data/test/test-diamonds.rb

@@ -0,0 +1,71 @@
+class DiamondsTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Diamonds.new
+  end
+
+  def record(*args)
+    Datasets::Diamonds::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   53940,
+                   {
+                     carat: 0.23,
+                     clarity: "SI2",
+                     color: "E",
+                     cut: "Ideal",
+                     depth: 61.5,
+                     price: 326,
+                     table: 55.0,
+                     x: 3.95,
+                     y: 3.98,
+                     z: 2.43,
+                   },
+                   {
+                     carat: 0.75,
+                     clarity: "SI2",
+                     color: "D",
+                     cut: "Ideal",
+                     depth: 62.2,
+                     price: 2757,
+                     table: 55.0,
+                     x: 5.83,
+                     y: 5.87,
+                     z: 3.64,
+                   },
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert_equal(<<-DESCRIPTION, description)
+Prices of over 50,000 round cut diamonds
+
+A dataset containing the prices and other attributes of almost 54,000
+ diamonds. The variables are as follows:
+
+A data frame with 53940 rows and 10 variables:
+
+  * price: price in US dollars ($326--$18,823)
+  * carat: weight of the diamond (0.2--5.01)
+  * cut: quality of the cut (Fair, Good, Very Good, Premium, Ideal)
+  * color: diamond colour, from D (best) to J (worst)
+  * clarity: a measurement of how clear the diamond is (I1 (worst), SI2,
+    SI1, VS2, VS1, VVS2, VVS1, IF (best))
+  * x: length in mm (0--10.74)
+  * y: width in mm (0--58.9)
+  * z: depth in mm (0--31.8)
+  * depth: total depth percentage = z / mean(x, y) = 2 * z / (x + y) (43--79)
+  * table: width of top of diamond relative to widest point (43--95)
+      DESCRIPTION
+    end
+  end
+end

data/test/test-fuel-economy.rb

@@ -0,0 +1,75 @@
+class FuelEconomyTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::FuelEconomy.new
+  end
+
+  def record(*args)
+    Datasets::FuelEconomy::Record.new(*args)
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+    assert_equal([
+                   234,
+                   {
+                      city_mpg: 18,
+                      displacement: 1.8,
+                      drive_train: "f",
+                      fuel: "p",
+                      highway_mpg: 29,
+                      manufacturer: "audi",
+                      model: "a4",
+                      n_cylinders: 4,
+                      transmission: "auto(l5)",
+                      type: "compact",
+                      year: 1999
+                   },
+                   {
+                      city_mpg: 17,
+                      displacement: 3.6,
+                      drive_train: "f",
+                      fuel: "p",
+                      highway_mpg: 26,
+                      manufacturer: "volkswagen",
+                      model: "passat",
+                      n_cylinders: 6,
+                      transmission: "auto(s6)",
+                      type: "midsize",
+                      year: 2008
+                    },
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert_equal(<<-DESCRIPTION, description)
+Fuel economy data from 1999 to 2008 for 38 popular models of cars
+
+This dataset contains a subset of the fuel economy data that the EPA makes
+available on https://fueleconomy.gov/. It contains only models which
+had a new release every year between 1999 and 2008 - this was used as a
+proxy for the popularity of the car.
+
+A data frame with 234 rows and 11 variables:
+
+  * manufacturer: manufacturer name
+  * model: model name
+  * displacement: engine displacement, in litres
+  * year: year of manufacture
+  * n_cylinders: number of cylinders
+  * transmissions: type of transmission
+  * drive_train: the type of drive train, where f = front-wheel drive, r = rear wheel drive, 4 = 4wd
+  * city_mpg: city miles per gallon
+  * highway_mpg: highway miles per gallon
+  * fuel: fuel type
+  * type: "type" of car
+      DESCRIPTION
+    end
+  end
+end

data/test/test-geolonia.rb

@@ -0,0 +1,64 @@
+class GeoloniaTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Geolonia.new
+  end
+
+  test('#each') do
+    records = @dataset.each.to_a
+    assert_equal([
+                   277191,
+                   {
+                     :prefecture_code => "01",
+                     :prefecture_name => "北海道",
+                     :prefecture_kana => "ホッカイドウ",
+                     :prefecture_romaji => "HOKKAIDO",
+                     :municipality_code => "01101",
+                     :municipality_name => "札幌市中央区",
+                     :municipality_kana => "サッポロシチュウオウク",
+                     :municipality_romaji => "SAPPORO SHI CHUO KU",
+                     :street_name => "旭ケ丘一丁目",
+                     :street_kana => "アサヒガオカ 1",
+                     :street_romaji => "ASAHIGAOKA 1",
+                     :alias => nil,
+                     :latitude => "43.04223",
+                     :longitude => "141.319722"
+                   },
+                   {
+                     :prefecture_code => "47",
+                     :prefecture_name => "沖縄県",
+                     :prefecture_kana => "オキナワケン",
+                     :prefecture_romaji => "OKINAWA KEN",
+                     :municipality_code => "47325",
+                     :municipality_name => "中頭郡嘉手納町",
+                     :municipality_kana => "ナカガミグンカデナチョウ",
+                     :municipality_romaji => "NAKAGAMI GUN KADENA CHO",
+                     :street_name => "字兼久",
+                     :street_kana => nil,
+                     :street_romaji => nil,
+                     :alias => "下原",
+                     :latitude => "26.351841",
+                     :longitude => "127.744975",
+                   },
+                 ],
+                 [
+                   records.size,
+                   records[0].to_h,
+                   records[-1].to_h,
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert_equal([
+                     "# Geolonia 住所データ",
+                     "## 住所データ仕様",
+                     "### ファイルフォーマット",
+                     "### 列",
+                   ],
+                   description.scan(/^#.*$/),
+                   description)
+    end
+  end
+
+end