red-datasets 0.1.4 → 0.1.6

data/test/test-wikipedia.rb

@@ -1,100 +1,54 @@
 class WikipediaTest < Test::Unit::TestCase
-  sub_test_case("ja") do
+  sub_test_case("en") do
     sub_test_case("articles") do
-      include Helper::Sandbox
-
       def setup
-        setup_sandbox
-        @dataset = Datasets::Wikipedia.new(language: :ja,
+        @dataset = Datasets::Wikipedia.new(language: :en,
                                            type: :articles)
-        def @dataset.cache_dir_path
-          @cache_dir_path
-        end
-        def @dataset.cache_dir_path=(path)
-          @cache_dir_path = path
-        end
-        @dataset.cache_dir_path = @tmp_dir
-      end
-
-      def teardown
-        teardown_sandbox
       end
 
       test("#each") do
-        def @dataset.download(output_path, url)
-          xml_path = output_path.sub_ext("")
-          xml_path.open("w") do |xml_file|
-            xml_file.puts(<<-XML)
-<mediawiki
-   xmlns="http://www.mediawiki.org/xml/export-0.10/"
-   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-   xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd"
-   version="0.10" xml:lang="ja">
-  <siteinfo>
-    <sitename>Wikipedia</sitename>
-  </siteinfo>
-  <page>
-    <title>タイトル</title>
-    <ns>4</ns>
-    <id>1</id>
-    <restrictions>sysop</restrictions>
-    <revision>
-      <id>3</id>
-      <parentid>2</parentid>
-      <timestamp>2004-04-30T14:46:00Z</timestamp>
-      <contributor>
-        <username>user</username>
-        <id>10</id>
-      </contributor>
-      <minor />
-      <comment>コメント</comment>
-      <model>wikitext</model>
-      <format>text/x-wiki</format>
-      <text xml:space="preserve">テキスト</text>
-      <sha1>a9674b19f8c56f785c91a555d0a144522bb318e6</sha1>
-    </revision>
-  </page>
-</mediawiki>
-            XML
-          end
-          unless system("bzip2", xml_path.to_s)
-            raise "failed to run bzip2"
-          end
-        end
-
-        contributor = Datasets::Wikipedia::Contributor.new("user", 10)
+        contributor = Datasets::Wikipedia::Contributor.new("Elli", 20842734)
         revision = Datasets::Wikipedia::Revision.new
-        revision.id = 3
-        revision.parent_id = 2
-        revision.timestamp = Time.iso8601("2004-04-30T14:46:00Z")
+        revision.id = 1002250816
+        revision.parent_id = 854851586
+        revision.timestamp = Time.iso8601("2021-01-23T15:15:01Z")
         revision.contributor = contributor
-        revision.comment = "コメント"
+        revision.comment = "shel"
         revision.model = "wikitext"
         revision.format = "text/x-wiki"
-        revision.text = "テキスト"
-        revision.sha1 = "a9674b19f8c56f785c91a555d0a144522bb318e6"
+        revision.text = <<-TEXT.chomp
+#REDIRECT [[Computer accessibility]]
+
+{{rcat shell|
+{{R from move}}
+{{R from CamelCase}}
+{{R unprintworthy}}
+}}
+        TEXT
+        revision.sha1 = "kmysdltgexdwkv2xsml3j44jb56dxvn"
         page = Datasets::Wikipedia::Page.new
-        page.title = "タイトル"
-        page.namespace = 4
-        page.id = 1
-        page.restrictions = ["sysop"]
+        page.title = "AccessibleComputing"
+        page.namespace = 0
+        page.id = 10
+        page.restrictions = nil
+        page.redirect = "Computer accessibility"
         page.revision = revision
         assert_equal(page, @dataset.each.first)
       end
 
       sub_test_case("#metadata") do
         test("#id") do
-          assert_equal("wikipedia-ja-articles",
+          assert_equal("wikipedia-en-articles",
                        @dataset.metadata.id)
         end
 
         test("#name") do
-          assert_equal("Wikipedia articles (ja)",
+          assert_equal("Wikipedia articles (en)",
                        @dataset.metadata.name)
         end
 
         test("#description") do
-          assert_equal("Wikipedia articles in ja",
+          assert_equal("Wikipedia articles in en",
                        @dataset.metadata.description)
         end
       end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - tomisuker
 - Kouhei Sutou
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.5
+        version: 3.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.5
+        version: 3.2.4
 - !ruby/object:Gem::Dependency
   name: rexml
   requirement: !ruby/object:Gem::Requirement
@@ -142,65 +142,98 @@ files:
 - doc/text/news.md
 - lib/datasets.rb
 - lib/datasets/adult.rb
+- lib/datasets/afinn.rb
+- lib/datasets/aozora-bunko.rb
+- lib/datasets/cache-path.rb
+- lib/datasets/california-housing.rb
 - lib/datasets/cifar.rb
 - lib/datasets/cldr-plurals.rb
 - lib/datasets/communities.rb
 - lib/datasets/dataset.rb
+- lib/datasets/diamonds.rb
 - lib/datasets/dictionary.rb
 - lib/datasets/downloader.rb
 - lib/datasets/e-stat-japan.rb
 - lib/datasets/error.rb
 - lib/datasets/fashion-mnist.rb
+- lib/datasets/fuel-economy.rb
+- lib/datasets/geolonia.rb
+- lib/datasets/ggplot2-dataset.rb
 - lib/datasets/hepatitis.rb
 - lib/datasets/iris.rb
+- lib/datasets/ita-corpus.rb
+- lib/datasets/kuzushiji-mnist.rb
+- lib/datasets/lazy.rb
 - lib/datasets/libsvm-dataset-list.rb
 - lib/datasets/libsvm.rb
+- lib/datasets/license.rb
+- lib/datasets/livedoor-news.rb
 - lib/datasets/metadata.rb
 - lib/datasets/mnist.rb
 - lib/datasets/mushroom.rb
+- lib/datasets/nagoya-university-conversation-corpus.rb
 - lib/datasets/penguins.rb
 - lib/datasets/penn-treebank.rb
+- lib/datasets/pmjt-dataset-list.rb
 - lib/datasets/postal-code-japan.rb
-- lib/datasets/rdatasets.rb
-- lib/datasets/seaborn-data.rb
+- lib/datasets/quora-duplicate-question-pair.rb
+- lib/datasets/rdataset.rb
+- lib/datasets/seaborn.rb
 - lib/datasets/sudachi-synonym-dictionary.rb
 - lib/datasets/table.rb
 - lib/datasets/tar-gz-readable.rb
 - lib/datasets/version.rb
+- lib/datasets/wikipedia-kyoto-japanese-english.rb
 - lib/datasets/wikipedia.rb
 - lib/datasets/wine.rb
+- lib/datasets/zip-extractor.rb
 - red-datasets.gemspec
 - test/helper.rb
 - test/run-test.rb
 - test/test-adult.rb
+- test/test-afinn.rb
+- test/test-aozora-bunko.rb
+- test/test-california-housing.rb
 - test/test-cifar.rb
 - test/test-cldr-plurals.rb
 - test/test-communities.rb
 - test/test-dataset.rb
+- test/test-diamonds.rb
 - test/test-dictionary.rb
 - test/test-downloader.rb
 - test/test-e-stat-japan.rb
 - test/test-fashion-mnist.rb
+- test/test-fuel-economy.rb
+- test/test-geolonia.rb
 - test/test-hepatitis.rb
 - test/test-iris.rb
+- test/test-ita-corpus.rb
+- test/test-kuzushiji-mnist.rb
 - test/test-libsvm-dataset-list.rb
 - test/test-libsvm.rb
+- test/test-license.rb
+- test/test-livedoor-news.rb
+- test/test-metadata.rb
 - test/test-mnist.rb
 - test/test-mushroom.rb
+- test/test-nagoya-university-conversation-corpus.rb
 - test/test-penguins.rb
 - test/test-penn-treebank.rb
+- test/test-pmjt-dataset-list.rb
 - test/test-postal-code-japan.rb
-- test/test-rdatasets.rb
-- test/test-seaborn-data.rb
+- test/test-quora-duplicate-question-pair.rb
+- test/test-rdataset.rb
+- test/test-seaborn.rb
 - test/test-sudachi-synonym-dictionary.rb
 - test/test-table.rb
+- test/test-wikipedia-kyoto-japanese-english.rb
 - test/test-wikipedia.rb
 - test/test-wine.rb
 homepage: https://github.com/red-data-tools/red-datasets
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -215,34 +248,49 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.0.dev
-signing_key: 
+rubygems_version: 3.5.0.dev
+signing_key:
 specification_version: 4
 summary: Red Datasets provides classes that provide common datasets such as iris dataset.
 test_files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-adult.rb
+- test/test-afinn.rb
+- test/test-aozora-bunko.rb
+- test/test-california-housing.rb
 - test/test-cifar.rb
 - test/test-cldr-plurals.rb
 - test/test-communities.rb
 - test/test-dataset.rb
+- test/test-diamonds.rb
 - test/test-dictionary.rb
 - test/test-downloader.rb
 - test/test-e-stat-japan.rb
 - test/test-fashion-mnist.rb
+- test/test-fuel-economy.rb
+- test/test-geolonia.rb
 - test/test-hepatitis.rb
 - test/test-iris.rb
+- test/test-ita-corpus.rb
+- test/test-kuzushiji-mnist.rb
 - test/test-libsvm-dataset-list.rb
 - test/test-libsvm.rb
+- test/test-license.rb
+- test/test-livedoor-news.rb
+- test/test-metadata.rb
 - test/test-mnist.rb
 - test/test-mushroom.rb
+- test/test-nagoya-university-conversation-corpus.rb
 - test/test-penguins.rb
 - test/test-penn-treebank.rb
+- test/test-pmjt-dataset-list.rb
 - test/test-postal-code-japan.rb
-- test/test-rdatasets.rb
-- test/test-seaborn-data.rb
+- test/test-quora-duplicate-question-pair.rb
+- test/test-rdataset.rb
+- test/test-seaborn.rb
 - test/test-sudachi-synonym-dictionary.rb
 - test/test-table.rb
+- test/test-wikipedia-kyoto-japanese-english.rb
 - test/test-wikipedia.rb
 - test/test-wine.rb

data/lib/datasets/seaborn-data.rb

@@ -1,49 +0,0 @@
-module Datasets
-  class SeabornData < Dataset
-    URL_FORMAT = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/%{name}.csv".freeze
-
-    def initialize(name)
-      super()
-      @metadata.id = "seaborn-data-#{name}"
-      @metadata.name = "SeabornData: #{name}"
-      @metadata.url = URL_FORMAT % {name: name}
-
-      @data_path = cache_dir_path + (name + ".csv")
-      @name = name
-    end
-
-    def each(&block)
-      return to_enum(__method__) unless block_given?
-
-      download(@data_path, @metadata.url) unless @data_path.exist?
-      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
-        csv.each do |row|
-          record = prepare_record(row)
-          yield record
-        end
-      end
-    end
-
-    private
-    def prepare_record(csv_row)
-      record = csv_row.to_h
-      record.transform_keys!(&:to_sym)
-
-      # Perform the same preprocessing as seaborn's load_dataset function
-      preprocessor = :"preprocess_#{@name}_record"
-      __send__(preprocessor, record) if respond_to?(preprocessor, true)
-
-      record
-    end
-
-    # The same preprocessing as seaborn.load_dataset
-    def preprocess_flights_record(record)
-      record[:month] &&= record[:month][0,3]
-    end
-
-    # The same preprocessing as seaborn.load_dataset
-    def preprocess_penguins_record(record)
-      record[:sex] &&= record[:sex].capitalize
-    end
-  end
-end

data/test/test-rdatasets.rb

@@ -1,136 +0,0 @@
-class RdatasetsTest < Test::Unit::TestCase
-  sub_test_case("RdatasetsList") do
-    def setup
-      @dataset = Datasets::RdatasetsList.new
-    end
-
-    sub_test_case("#each") do
-      test("with package_name") do
-        records = @dataset.filter(package: "datasets").to_a
-        assert_equal([
-                       84,
-                       {
-                         package: "datasets",
-                         dataset: "ability.cov",
-                         title: "Ability and Intelligence Tests",
-                         rows: 6,
-                         cols: 8,
-                         n_binary: 0,
-                         n_character: 0,
-                         n_factor: 0,
-                         n_logical: 0,
-                         n_numeric: 8,
-                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/ability.cov.csv",
-                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/ability.cov.html"
-                       },
-                       {
-                         package: "datasets",
-                         dataset: "WWWusage",
-                         title: "Internet Usage per Minute",
-                         rows: 100,
-                         cols: 2,
-                         n_binary: 0,
-                         n_character: 0,
-                         n_factor: 0,
-                         n_logical: 0,
-                         n_numeric: 2,
-                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/datasets/WWWusage.csv",
-                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/datasets/WWWusage.html"
-                       }
-                     ],
-                     [
-                       records.size,
-                       records[0].to_h,
-                       records[-1].to_h
-                     ])
-      end
-
-      test("without package_name") do
-        records = @dataset.each.to_a
-        assert_equal([
-                       1714,
-                       {
-                         package: "AER",
-                         dataset: "Affairs",
-                         title: "Fair's Extramarital Affairs Data",
-                         rows: 601,
-                         cols: 9,
-                         n_binary: 2,
-                         n_character: 0,
-                         n_factor: 2,
-                         n_logical: 0,
-                         n_numeric: 7,
-                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/AER/Affairs.csv",
-                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/AER/Affairs.html"
-                       },
-                       {
-                         package: "vcd",
-                         dataset: "WomenQueue",
-                         title: "Women in Queues",
-                         rows: 11,
-                         cols: 2,
-                         n_binary: 0,
-                         n_character: 0,
-                         n_factor: 1,
-                         n_logical: 0,
-                         n_numeric: 1,
-                         csv: "https://vincentarelbundock.github.io/Rdatasets/csv/vcd/WomenQueue.csv",
-                         doc: "https://vincentarelbundock.github.io/Rdatasets/doc/vcd/WomenQueue.html"
-                       },
-                     ],
-                     [
-                       records.size,
-                       records[0].to_h,
-                       records[-1].to_h
-                     ])
-      end
-    end
-  end
-
-  sub_test_case("Rdatasets") do
-    sub_test_case("datasets") do
-      sub_test_case("AirPassengers") do
-        def setup
-          @dataset = Datasets::Rdatasets.new("datasets", "AirPassengers")
-        end
-
-        test("#each") do
-          records = @dataset.each.to_a
-          assert_equal([
-                         144,
-                         { time: 1949,             value: 112 },
-                         { time: 1960.91666666667, value: 432 },
-                       ],
-                       [
-                         records.size,
-                         records[0],
-                         records[-1]
-                       ])
-        end
-
-        test("#metadata.id") do
-          assert_equal("rdatasets-datasets-AirPassengers", @dataset.metadata.id)
-        end
-
-        test("#metadata.description") do
-          description = @dataset.metadata.description
-          assert do
-            description.include?("Monthly Airline Passenger Numbers 1949-1960")
-          end
-        end
-      end
-
-      test("invalid dataset name") do
-        assert_raise(ArgumentError) do
-          Datasets::Rdatasets.new("datasets", "invalid datasets name")
-        end
-      end
-    end
-
-    test("invalid package name") do
-      assert_raise(ArgumentError) do
-        Datasets::Rdatasets.new("invalid package name", "AirPassengers")
-      end
-    end
-  end
-end