red-datasets 0.1.3 → 0.1.5

data/test/test-ita-corpus.rb

@@ -0,0 +1,69 @@
+class ITACorpusTest < Test::Unit::TestCase
+
+  sub_test_case("type") do  
+    test("emotion") do
+      dataset = Datasets::ITACorpus.new(type: :emotion)
+      records = dataset.to_a
+      assert_equal([
+                    100,
+                    {
+                      :id => "EMOTION100_001",
+                      :sentence => "えっ嘘でしょ。,エッウソデショ。"
+                    },
+                    {
+                      :id => "EMOTION100_100",
+                      :sentence => "ラーテャン。,ラーテャン。",
+                    },
+                  ],
+                  [
+                    records.size,
+                    records[0].to_h,
+                    records[-1].to_h,
+                  ])
+    end
+
+    test("recitation") do
+      dataset = Datasets::ITACorpus.new(type: :recitation)
+      records = dataset.to_a
+      assert_equal([
+                    324,
+                    {
+                      :id => "RECITATION324_001",
+                      :sentence => "女の子がキッキッ嬉しそう。,オンナノコガキッキッウレシソー。"
+                    },
+                    {
+                      :id => "RECITATION324_324",
+                      :sentence => "チュクンの波長は、パツンと共通している。,チュクンノハチョーワ、パツントキョーツウシテイル。",
+                    },
+                  ],
+                  [
+                    records.size,
+                    records[0].to_h,
+                    records[-1].to_h,
+                  ])
+    end
+
+    test("invalid") do
+      message = "Please set type :emotion or :recitation: :invalid"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::ITACorpus.new(type: :invalid)
+      end
+    end
+
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      dataset = Datasets::ITACorpus.new(type: :emotion)
+      description = dataset.metadata.description
+      assert_equal([
+                     "# ITAコーパスの文章リスト公開用リポジトリ",
+                     "## ITAコーパスとは",
+                     "## ITAコーパスの文献情報"
+                   ],
+                   description.scan(/^#.*$/),
+                   description)
+    end
+  end
+
+end

data/test/test-kuzushiji-mnist.rb

@@ -0,0 +1,137 @@
+class KuzushijiMNISTTest < Test::Unit::TestCase
+  sub_test_case("Normal") do
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::KuzushijiMNIST.new(type: :train)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       60000,
+                       [
+                         8,
+                         784,
+                         [213, 233, 255, 186, 2, 0, 0, 0, 0, 0],
+                         [0, 0, 0, 0, 0, 0, 0, 0, 45, 252],
+                       ],
+                       [
+                         9,
+                         784,
+                         [81, 246, 254, 155, 224, 255, 230, 39, 0, 0],
+                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [213, 233, 255, 186, 2, 0, 0, 0, 0, 0],
+                       [81, 246, 254, 155, 224, 255, 230, 39, 0, 0],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("kuzushiji-mnist-train", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Kuzushiji-MNIST: train", @dataset.metadata.name)
+        end
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::KuzushijiMNIST.new(type: :test)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       10000,
+                       [
+                         2,
+                         784,
+                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 75],
+                         [44, 255, 255, 246, 119, 252, 46, 0, 70, 255],
+                       ],
+                       [
+                         2,
+                         784,
+                         [0, 0, 0, 0, 0, 0, 0, 84, 255, 192],
+                         [0, 0, 0, 0, 0, 23, 245, 92, 42, 254],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [0, 0, 0, 0, 0, 0, 0, 0, 0, 75],
+                       [0, 0, 0, 0, 0, 0, 0, 84, 255, 192],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("kuzushiji-mnist-test", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Kuzushiji-MNIST: test", @dataset.metadata.name)
+        end
+      end
+    end
+  end
+
+  sub_test_case("Abnormal") do
+    test("invalid type") do
+      invalid_type = :invalid
+      message = "Please set type :train or :test: #{invalid_type.inspect}"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::KuzushijiMNIST.new(type: invalid_type)
+      end
+    end
+  end
+end

data/test/test-license.rb

@@ -0,0 +1,24 @@
+class LicenseTest < Test::Unit::TestCase
+  sub_test_case(".try_convert") do
+    test("String") do
+      assert_equal(Datasets::License.new("Apache-2.0"),
+                   Datasets::License.try_convert("Apache-2.0"))
+    end
+
+    test("{spdx_id:}") do
+      assert_equal(Datasets::License.new("Apache-2.0"),
+                   Datasets::License.try_convert(spdx_id: "Apache-2.0"))
+    end
+
+    test("{name:, url:}") do
+      license = {
+        name: "Quora's Terms of Service",
+        url: "https://www.quora.com/about/tos",
+      }
+      assert_equal(Datasets::License.new(nil,
+                                         "Quora's Terms of Service",
+                                         "https://www.quora.com/about/tos"),
+                   Datasets::License.try_convert(license))
+    end
+  end
+end

data/test/test-livedoor-news.rb

@@ -0,0 +1,351 @@
+class LivedoorNewsTest < Test::Unit::TestCase
+  sub_test_case("type") do
+    test("topic_news") do
+      dataset = Datasets::LivedoorNews.new(type: :topic_news)
+      records = dataset.to_a
+      assert_equal([
+                     770,
+                     [
+                       "http://news.livedoor.com/article/detail/5903225/",
+                       Time.iso8601("2011-10-02T10:00:00+0900"),
+                       "悪評が次から次へと溢",
+                       "/5571502/\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6918105/",
+                       Time.iso8601("2012-09-04T12:45:00+0900"),
+                       "ジャンプ連載漫画が終",
+                       "提案も散見された。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("sports_watch") do
+      dataset = Datasets::LivedoorNews.new(type: :sports_watch)
+      records = dataset.to_a
+      assert_equal([
+                     900,
+                     [
+                       "http://news.livedoor.com/article/detail/4597641/",
+                       Time.iso8601("2010-02-10T10:50:00+0900"),
+                       "【Sports Wa",
+                       "送る秋山であった。\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6917848/",
+                       Time.iso8601("2012-09-04T11:25:00+0900"),
+                       "ジーコ、本田圭佑につ",
+                       "る」と語っている。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("it_life_hack") do
+      dataset = Datasets::LivedoorNews.new(type: :it_life_hack)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/6292880/",
+                       Time.iso8601("2012-02-19T13:00:00+0900"),
+                       "旧式Macで禁断のパ",
+                       "p\n" + "クチコミを見る\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6918825/",
+                       Time.iso8601("2012-09-04T15:00:00+0900"),
+                       "レノボWindows",
+                       "J\n" + "クチコミを見る\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("kaden_channel") do
+      dataset = Datasets::LivedoorNews.new(type: :kaden_channel)
+      records = dataset.to_a
+      assert_equal([
+                     864,
+                     [
+                       "http://news.livedoor.com/article/detail/5774093/",
+                       Time.iso8601("2011-08-10T10:00:00+0900"),
+                       "【ニュース】電力使用",
+                       "に備える【デジ通】\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6919353/",
+                       Time.iso8601("2012-09-04T17:00:00+0900"),
+                       "Hulu、ついに待望",
+                       "uに今後も注目だ。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("movie_enter") do
+      dataset = Datasets::LivedoorNews.new(type: :movie_enter)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/5840081/",
+                       Time.iso8601("2011-09-08T10:00:00+0900"),
+                       "インタビュー：宮崎あ",
+                       "ない。 1 2 3\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6909318/",
+                       Time.iso8601("2012-09-01T10:30:00+0900"),
+                       "【週末映画まとめ読み",
+                       "レイ+DVDセット\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("dokujo_tsushin") do
+      dataset = Datasets::LivedoorNews.new(type: :dokujo_tsushin)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/4778030/",
+                       Time.iso8601("2010-05-22T14:30:00+0900"),
+                       "友人代表のスピーチ、",
+                       "も幸あれ（高山惠）\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6915005/",
+                       Time.iso8601("2012-09-03T14:00:00+0900"),
+                       "男女間で“カワイイ”",
+                       "ツー／神田はるひ）\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("smax") do
+      dataset = Datasets::LivedoorNews.new(type: :smax)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/6507397/",
+                       Time.iso8601("2012-04-26T16:55:00+0900"),
+                       "あのアプリもこのアプ",
+                       "n Twitter\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6919324/",
+                       Time.iso8601("2012-09-04T16:55:00+0900"),
+                       "【究極にカスタマイズ",
+                       "個人） : 富士通\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("livedoor_homme") do
+      dataset = Datasets::LivedoorNews.new(type: :livedoor_homme)
+      records = dataset.to_a
+      assert_equal([
+                     511,
+                     [
+                       "http://news.livedoor.com/article/detail/4568088/",
+                       Time.iso8601("2010-01-24T18:10:00+0900"),
+                       "フォーエバー２１旗艦",
+                       "フォーエバー２１」\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6828491/",
+                       Time.iso8601("2012-08-06T14:30:00+0900"),
+                       "【女子座談会】ぶっち",
+                       "タートキャンペーン\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("peachy") do
+      dataset = Datasets::LivedoorNews.new(type: :peachy)
+      records = dataset.to_a
+      assert_equal([
+                     842,
+                     [
+                       "http://news.livedoor.com/article/detail/4289213/",
+                       Time.iso8601("2009-08-07T20:30:00+0900"),
+                       "韓国の伝統菓子を食べ",
+                       "試してみませんか？\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6908055/",
+                       Time.iso8601("2012-09-01T18:00:00+0900"),
+                       "初デートで彼を悶絶さ",
+                       "hyアプリの使い方\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("invalid") do
+      message = ":type must be one of [" +
+                ":topic_news, " +
+                ":sports_watch, " +
+                ":it_life_hack, " +
+                ":kaden_channel, " +
+                ":movie_enter, " +
+                ":dokujo_tsushin, " +
+                ":smax, " +
+                ":livedoor_homme, " +
+                ":peachy" +
+                "]: :invalid"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::LivedoorNews.new(type: :invalid)
+      end
+    end
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      dataset = Datasets::LivedoorNews.new(type: :topic_news)
+      description = dataset.metadata.description
+      assert_equal([
+                     "livedoor ニ",
+                     "に感謝いたします。\n"
+                   ],
+                   [
+                     description[0,10],
+                     description[-10,10]
+                   ],
+                   description)
+    end
+  end
+end

data/test/test-metadata.rb

@@ -0,0 +1,36 @@
+class MetadataTest < Test::Unit::TestCase
+  def setup
+    @metadata = Datasets::Metadata.new
+  end
+
+  sub_test_case("#licenses") do
+    test("String") do
+      @metadata.licenses = "Apache-2.0"
+      assert_equal([Datasets::License.new("Apache-2.0")],
+                   @metadata.licenses)
+    end
+
+    test("[String]") do
+      @metadata.licenses = ["Apache-2.0"]
+      assert_equal([Datasets::License.new("Apache-2.0")],
+                   @metadata.licenses)
+    end
+
+    test("{name:, url:}") do
+      @metadata.licenses = {
+        name: "Quora's Terms of Service",
+        url: "https://www.quora.com/about/tos",
+      }
+      assert_equal([Datasets::License.new(nil,
+                                          "Quora's Terms of Service",
+                                          "https://www.quora.com/about/tos")],
+                   @metadata.licenses)
+    end
+
+    test("Symbol") do
+      assert_raise(ArgumentError.new("invalid license: :apache_2_0")) do
+        @metadata.licenses = :apache_2_0
+      end
+    end
+  end
+end

data/test/test-penguins.rb

@@ -7,7 +7,7 @@ class PenguinsTest < Test::Unit::TestCase
                        dataset = cls.new
                        dataset.data_path.relative_path_from(dataset.send(:cache_dir_path)).to_s
                      }
-      assert_equal(["penguins/adelie.csv", "penguins/gentoo.csv", "penguins/chinstrap.csv"],
+      assert_equal(["adelie.csv", "gentoo.csv", "chinstrap.csv"],
                    data_paths)
     end
   end

data/test/test-pmjt-dataset-list.rb

@@ -0,0 +1,50 @@
+class PMJTDatasetListTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets:: PMJTDatasetList.new
+  end
+
+  test("#each") do
+    records = @dataset.each.to_a
+
+    record_first = Datasets::PMJTDatasetList::Record.new
+    record_first.unit = '冊'
+    record_first.open_data_category = '総記'
+    record_first.tag = nil
+    record_first.release_time = 'H31.1'
+    record_first.n_volumes = '2'
+    record_first.type = '刊'
+    record_first.publication_year = '元禄９'
+    record_first.original_request_code = '９９－３７－１～２'
+    record_first.id = '200003090'
+    record_first.title = '人倫重宝記'
+    record_first.text = nil
+    record_first.bibliographical_introduction = nil
+    record_first.year = nil
+
+    record_last = Datasets::PMJTDatasetList::Record.new
+    record_last.unit = '冊'
+    record_last.open_data_category = '総記'
+    record_last.tag = nil
+    record_last.release_time = 'H27.11'
+    record_last.n_volumes = '1'
+    record_last.type = '刊'
+    record_last.publication_year = '慶応2'
+    record_last.original_request_code = '４９－１７３'
+    record_last.id = '200021837'
+    record_last.title = '洋学便覧'
+    record_last.text = nil
+    record_last.bibliographical_introduction = '○'
+    record_last.year = '1866'
+
+    assert_equal([
+                    3126,
+                    record_first,
+                    record_last
+                  ],
+                  [
+                    records.size,
+                    records[1],
+                    records[-1]
+                  ])
+  end
+end