red-datasets 0.1.4 → 0.1.6

data/test/test-kuzushiji-mnist.rb

@@ -0,0 +1,137 @@
+class KuzushijiMNISTTest < Test::Unit::TestCase
+  sub_test_case("Normal") do
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::KuzushijiMNIST.new(type: :train)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       60000,
+                       [
+                         8,
+                         784,
+                         [213, 233, 255, 186, 2, 0, 0, 0, 0, 0],
+                         [0, 0, 0, 0, 0, 0, 0, 0, 45, 252],
+                       ],
+                       [
+                         9,
+                         784,
+                         [81, 246, 254, 155, 224, 255, 230, 39, 0, 0],
+                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [213, 233, 255, 186, 2, 0, 0, 0, 0, 0],
+                       [81, 246, 254, 155, 224, 255, 230, 39, 0, 0],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("kuzushiji-mnist-train", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Kuzushiji-MNIST: train", @dataset.metadata.name)
+        end
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::KuzushijiMNIST.new(type: :test)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       10000,
+                       [
+                         2,
+                         784,
+                         [0, 0, 0, 0, 0, 0, 0, 0, 0, 75],
+                         [44, 255, 255, 246, 119, 252, 46, 0, 70, 255],
+                       ],
+                       [
+                         2,
+                         784,
+                         [0, 0, 0, 0, 0, 0, 0, 84, 255, 192],
+                         [0, 0, 0, 0, 0, 23, 245, 92, 42, 254],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [0, 0, 0, 0, 0, 0, 0, 0, 0, 75],
+                       [0, 0, 0, 0, 0, 0, 0, 84, 255, 192],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("kuzushiji-mnist-test", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Kuzushiji-MNIST: test", @dataset.metadata.name)
+        end
+      end
+    end
+  end
+
+  sub_test_case("Abnormal") do
+    test("invalid type") do
+      invalid_type = :invalid
+      message = "Please set type :train or :test: #{invalid_type.inspect}"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::KuzushijiMNIST.new(type: invalid_type)
+      end
+    end
+  end
+end

data/test/test-license.rb

@@ -0,0 +1,24 @@
+class LicenseTest < Test::Unit::TestCase
+  sub_test_case(".try_convert") do
+    test("String") do
+      assert_equal(Datasets::License.new("Apache-2.0"),
+                   Datasets::License.try_convert("Apache-2.0"))
+    end
+
+    test("{spdx_id:}") do
+      assert_equal(Datasets::License.new("Apache-2.0"),
+                   Datasets::License.try_convert(spdx_id: "Apache-2.0"))
+    end
+
+    test("{name:, url:}") do
+      license = {
+        name: "Quora's Terms of Service",
+        url: "https://www.quora.com/about/tos",
+      }
+      assert_equal(Datasets::License.new(nil,
+                                         "Quora's Terms of Service",
+                                         "https://www.quora.com/about/tos"),
+                   Datasets::License.try_convert(license))
+    end
+  end
+end

data/test/test-livedoor-news.rb

@@ -0,0 +1,351 @@
+class LivedoorNewsTest < Test::Unit::TestCase
+  sub_test_case("type") do
+    test("topic_news") do
+      dataset = Datasets::LivedoorNews.new(type: :topic_news)
+      records = dataset.to_a
+      assert_equal([
+                     770,
+                     [
+                       "http://news.livedoor.com/article/detail/5903225/",
+                       Time.iso8601("2011-10-02T10:00:00+0900"),
+                       "悪評が次から次へと溢",
+                       "/5571502/\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6918105/",
+                       Time.iso8601("2012-09-04T12:45:00+0900"),
+                       "ジャンプ連載漫画が終",
+                       "提案も散見された。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("sports_watch") do
+      dataset = Datasets::LivedoorNews.new(type: :sports_watch)
+      records = dataset.to_a
+      assert_equal([
+                     900,
+                     [
+                       "http://news.livedoor.com/article/detail/4597641/",
+                       Time.iso8601("2010-02-10T10:50:00+0900"),
+                       "【Sports Wa",
+                       "送る秋山であった。\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6917848/",
+                       Time.iso8601("2012-09-04T11:25:00+0900"),
+                       "ジーコ、本田圭佑につ",
+                       "る」と語っている。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("it_life_hack") do
+      dataset = Datasets::LivedoorNews.new(type: :it_life_hack)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/6292880/",
+                       Time.iso8601("2012-02-19T13:00:00+0900"),
+                       "旧式Macで禁断のパ",
+                       "p\n" + "クチコミを見る\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6918825/",
+                       Time.iso8601("2012-09-04T15:00:00+0900"),
+                       "レノボWindows",
+                       "J\n" + "クチコミを見る\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("kaden_channel") do
+      dataset = Datasets::LivedoorNews.new(type: :kaden_channel)
+      records = dataset.to_a
+      assert_equal([
+                     864,
+                     [
+                       "http://news.livedoor.com/article/detail/5774093/",
+                       Time.iso8601("2011-08-10T10:00:00+0900"),
+                       "【ニュース】電力使用",
+                       "に備える【デジ通】\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6919353/",
+                       Time.iso8601("2012-09-04T17:00:00+0900"),
+                       "Hulu、ついに待望",
+                       "uに今後も注目だ。\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("movie_enter") do
+      dataset = Datasets::LivedoorNews.new(type: :movie_enter)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/5840081/",
+                       Time.iso8601("2011-09-08T10:00:00+0900"),
+                       "インタビュー：宮崎あ",
+                       "ない。 1 2 3\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6909318/",
+                       Time.iso8601("2012-09-01T10:30:00+0900"),
+                       "【週末映画まとめ読み",
+                       "レイ+DVDセット\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("dokujo_tsushin") do
+      dataset = Datasets::LivedoorNews.new(type: :dokujo_tsushin)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/4778030/",
+                       Time.iso8601("2010-05-22T14:30:00+0900"),
+                       "友人代表のスピーチ、",
+                       "も幸あれ（高山惠）\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6915005/",
+                       Time.iso8601("2012-09-03T14:00:00+0900"),
+                       "男女間で“カワイイ”",
+                       "ツー／神田はるひ）\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("smax") do
+      dataset = Datasets::LivedoorNews.new(type: :smax)
+      records = dataset.to_a
+      assert_equal([
+                     870,
+                     [
+                       "http://news.livedoor.com/article/detail/6507397/",
+                       Time.iso8601("2012-04-26T16:55:00+0900"),
+                       "あのアプリもこのアプ",
+                       "n Twitter\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6919324/",
+                       Time.iso8601("2012-09-04T16:55:00+0900"),
+                       "【究極にカスタマイズ",
+                       "個人） : 富士通\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("livedoor_homme") do
+      dataset = Datasets::LivedoorNews.new(type: :livedoor_homme)
+      records = dataset.to_a
+      assert_equal([
+                     511,
+                     [
+                       "http://news.livedoor.com/article/detail/4568088/",
+                       Time.iso8601("2010-01-24T18:10:00+0900"),
+                       "フォーエバー２１旗艦",
+                       "フォーエバー２１」\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6828491/",
+                       Time.iso8601("2012-08-06T14:30:00+0900"),
+                       "【女子座談会】ぶっち",
+                       "タートキャンペーン\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("peachy") do
+      dataset = Datasets::LivedoorNews.new(type: :peachy)
+      records = dataset.to_a
+      assert_equal([
+                     842,
+                     [
+                       "http://news.livedoor.com/article/detail/4289213/",
+                       Time.iso8601("2009-08-07T20:30:00+0900"),
+                       "韓国の伝統菓子を食べ",
+                       "試してみませんか？\n"
+                     ],
+                     [
+                       "http://news.livedoor.com/article/detail/6908055/",
+                       Time.iso8601("2012-09-01T18:00:00+0900"),
+                       "初デートで彼を悶絶さ",
+                       "hyアプリの使い方\n"
+                     ],
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].url,
+                       records[0].timestamp,
+                       records[0].sentence[0, 10],
+                       records[0].sentence[-10, 10]
+                     ],
+                     [
+                       records[-1].url,
+                       records[-1].timestamp,
+                       records[-1].sentence[0, 10],
+                       records[-1].sentence[-10, 10]
+                     ],
+                   ])
+    end
+
+    test("invalid") do
+      message = ":type must be one of [" +
+                ":topic_news, " +
+                ":sports_watch, " +
+                ":it_life_hack, " +
+                ":kaden_channel, " +
+                ":movie_enter, " +
+                ":dokujo_tsushin, " +
+                ":smax, " +
+                ":livedoor_homme, " +
+                ":peachy" +
+                "]: :invalid"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::LivedoorNews.new(type: :invalid)
+      end
+    end
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      dataset = Datasets::LivedoorNews.new(type: :topic_news)
+      description = dataset.metadata.description
+      assert_equal([
+                     "livedoor ニ",
+                     "に感謝いたします。\n"
+                   ],
+                   [
+                     description[0,10],
+                     description[-10,10]
+                   ],
+                   description)
+    end
+  end
+end

data/test/test-metadata.rb

@@ -0,0 +1,36 @@
+class MetadataTest < Test::Unit::TestCase
+  def setup
+    @metadata = Datasets::Metadata.new
+  end
+
+  sub_test_case("#licenses") do
+    test("String") do
+      @metadata.licenses = "Apache-2.0"
+      assert_equal([Datasets::License.new("Apache-2.0")],
+                   @metadata.licenses)
+    end
+
+    test("[String]") do
+      @metadata.licenses = ["Apache-2.0"]
+      assert_equal([Datasets::License.new("Apache-2.0")],
+                   @metadata.licenses)
+    end
+
+    test("{name:, url:}") do
+      @metadata.licenses = {
+        name: "Quora's Terms of Service",
+        url: "https://www.quora.com/about/tos",
+      }
+      assert_equal([Datasets::License.new(nil,
+                                          "Quora's Terms of Service",
+                                          "https://www.quora.com/about/tos")],
+                   @metadata.licenses)
+    end
+
+    test("Symbol") do
+      assert_raise(ArgumentError.new("invalid license: :apache_2_0")) do
+        @metadata.licenses = :apache_2_0
+      end
+    end
+  end
+end

data/test/test-nagoya-university-conversation-corpus.rb

@@ -0,0 +1,132 @@
+class NagoyaUniversityConversationCorpusTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::NagoyaUniversityConversationCorpus.new
+  end
+
+  sub_test_case("each") do
+    test("#sentences") do
+      records = @dataset.each.to_a
+      first_sentences = records[0].sentences
+      last_sentences = records[-1].sentences
+      assert_equal([
+                     856,
+                     {
+                       participant_id: 'F107',
+                       content: '＊＊＊の町というのはちいちゃくって、城壁がこう町全体をぐるっと回ってて、それが城壁の上を歩いても１時間ぐらいですよね。'
+                     },
+                     {
+                       participant_id: nil,
+                       content: nil
+                     },
+                     603,
+                     {
+                       participant_id: 'F007',
+                       content: 'それでは話を始めまーす。'
+                     },
+                     {
+                       participant_id: nil,
+                       content: nil
+                     }
+                   ],
+                   [
+                     first_sentences.size,
+                     first_sentences[0].to_h,
+                     first_sentences[-1].to_h,
+                     last_sentences.size,
+                     last_sentences[0].to_h,
+                     last_sentences[-1].to_h,
+                   ])
+    end
+
+    test("#participants") do
+      records = @dataset.each.to_a
+      first_participants = records[0].participants
+      last_participants = records[-1].participants
+      assert_equal([
+                     4,
+                     {
+                       id: 'F107',
+                       attribute: '女性３０代後半',
+                       birthplace: '愛知県幡豆郡出身',
+                       residence: '愛知県幡豆郡在住'
+                     },
+                     {
+                       id: 'F128',
+                       attribute: '女性２０代前半',
+                       birthplace: '愛知県西尾市出身',
+                       residence: '西尾市在住'
+                     },
+                     2,
+                     {
+                       id: 'F007',
+                       attribute: '女性５０代後半',
+                       birthplace: '東京都出身',
+                       residence: '東京都国分寺市在住'
+                     },
+                     {
+                       id: 'F003',
+                       attribute: '女性８０代後半',
+                       birthplace: '栃木県宇都宮市出身',
+                       residence: '国分寺市在住'
+                     }
+                   ],
+                   [
+                     first_participants.size,
+                     first_participants[0].to_h,
+                     first_participants[-1].to_h,
+                     last_participants.size,
+                     last_participants[0].to_h,
+                     last_participants[-1].to_h
+                   ])
+    end
+
+    test("others") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     129,
+                     [
+                       '１（約３５分）',
+                       '２００１年１０月１６日',
+                       'ファミリーレストラン',
+                       '英会話教室の友人',
+                       nil
+                     ],
+                     [
+                       '１２９（３６分）',
+                       '２００３年２月１６日',
+                       '二人の自宅',
+                       '母と娘',
+                       'F007は東京に３８年、F003は東京に６０年居住。'
+                    ]
+                   ],
+                   [
+                     records.size,
+                     [
+                       records[0].name,
+                       records[0].date,
+                       records[0].place,
+                       records[0].relationships,
+                       records[0].note
+                     ],
+                     [
+                       records[-1].name,
+                       records[-1].date,
+                       records[-1].place,
+                       records[-1].relationships,
+                       records[-1].note
+                     ]
+                   ])
+    end
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert_equal(<<~DESCRIPTION, description)
+        The "Nagoya University Conversation Corpus" is a corpus of 129 conversations,
+        total about 100 hours of chatting among native speakers of Japanese,
+        which is converted into text.
+      DESCRIPTION
+    end
+  end
+end

data/test/test-penguins.rb

@@ -7,7 +7,7 @@ class PenguinsTest < Test::Unit::TestCase
                        dataset = cls.new
                        dataset.data_path.relative_path_from(dataset.send(:cache_dir_path)).to_s
                      }
-      assert_equal(["penguins/adelie.csv", "penguins/gentoo.csv", "penguins/chinstrap.csv"],
+      assert_equal(["adelie.csv", "gentoo.csv", "chinstrap.csv"],
                    data_paths)
     end
   end