red-datasets 0.0.6 → 0.1.1

data/test/test-dictionary.rb

@@ -0,0 +1,43 @@
+class DictionaryTest < Test::Unit::TestCase
+  def setup
+    penn_treebank = Datasets::PennTreebank.new(type: :test)
+    @dictionary = penn_treebank.to_table.dictionary_encode(:word)
+  end
+
+  test("#id") do
+    assert_equal(95, @dictionary.id("<unk>"))
+  end
+
+  test("#value") do
+    assert_equal("<unk>", @dictionary.value(95))
+  end
+
+  test("#ids") do
+    assert_equal([0, 1, 2, 3, 4], @dictionary.ids.first(5))
+  end
+
+  test("#values") do
+    assert_equal(["no", "it", "was", "n't", "black"],
+                 @dictionary.values.first(5))
+  end
+
+  test("#each") do
+    assert_equal([
+                   [0, "no"],
+                   [1, "it"],
+                   [2, "was"],
+                   [3, "n't"],
+                   [4, "black"],
+                 ],
+                 @dictionary.each.first(5).to_a)
+  end
+
+  test("#size") do
+    assert_equal(6048, @dictionary.size)
+  end
+
+  test("#length") do
+    assert_equal(@dictionary.size,
+                 @dictionary.length)
+  end
+end

data/test/test-e-stat-japan.rb

@@ -0,0 +1,383 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'tmpdir'
+
+class EStatJapanTest < Test::Unit::TestCase
+  sub_test_case('app_id') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('nothing') do
+      assert_raise(Datasets::EStatJapan::ArgumentError) do
+        Datasets::EStatJapan::StatsData.new('test-data-id')
+      end
+    end
+
+    test('constructor') do
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+
+    test('env') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_env', stats_data.app_id)
+    end
+
+    test('configure') do
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure & constructor') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+  end
+
+  sub_test_case('url generation') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('generates url correctly') do
+      Datasets::EStatJapan.app_id = 'abcdef'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      stats_data_id = '000000'
+      stats_data.instance_eval do
+        @id = stats_data_id
+        @base_url = 'http://testurl/rest/2.1/app/json/getStatsData'
+      end
+      url = stats_data.send(:generate_url)
+      assert_equal(
+        'http://testurl/rest/2.1/app/json/getStatsData' \
+        '?appId=abcdef&lang=J&statsDataId=000000&' \
+        'metaGetFlg=Y&cntGetFlg=N&sectionHeaderFlg=1',
+        url.to_s
+      )
+    end
+  end
+
+  sub_test_case('parsing records') do
+    def setup
+      Datasets::EStatJapan.app_id = nil
+      # prepare test data
+      class_obj = [
+        {
+          "@name": 'table1',
+          "@id": 'tab',
+          "CLASS": {
+            "@level": '1',
+            "@code": '00001',
+            "@name": 'table1'
+          }
+        },
+        {
+          "@name": 'data1',
+          "@id": 'cat01',
+          "CLASS": {
+            "@level": '1',
+            "@code": 'data1',
+            "@name": 'data1_name'
+          }
+        },
+        {
+          "@name": 'area1',
+          "@id": 'area',
+          "CLASS": [
+            {
+              "@level": '2',
+              "@code": '01100',
+              "@name": 'test1 big-city',
+              "@parentCode": '01000'
+            },
+            {
+              "@level": '3',
+              "@code": '01101',
+              "@name": 'test1 big-city a-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '3',
+              "@code": '01102',
+              "@name": 'test1 big-city b-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '2',
+              "@code": '02555',
+              "@name": 'test2 a-city',
+              "@parentCode": '02000'
+            },
+            {
+              "@level": '2',
+              "@code": '02556',
+              "@name": 'test2 b-city',
+              "@parentCode": '02000'
+            }
+          ]
+        },
+        {
+          "@name": 'time',
+          "@id": 'time',
+          "CLASS": [
+            {
+              "@level": '1',
+              "@code": 'time1',
+              "@name": 'time1'
+            },
+            {
+              "@level": '1',
+              "@code": 'time2',
+              "@name": 'time2'
+            },
+            {
+              "@level": '1',
+              "@code": 'time3',
+              "@name": 'time3'
+            }
+          ]
+        }
+      ]
+      data_inf = class_obj[2][:CLASS].map do |entry|
+        [
+          {
+            "$": 1000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time1',
+            "@unit": 'person'
+          },
+          {
+            "$": 2000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time2',
+            "@unit": 'person'
+          }
+        ]
+      end.flatten
+      ## test record for `skip_nil_row: true`
+      data_inf << {
+        "$": 3000,
+        "@area": '02556',
+        "@cat01": 'data1',
+        "@tab": 'table1',
+        "@time": 'time3',
+        "@unit": 'person'
+      }
+      @response_data_default = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 0,
+            'ERROR_MSG' => 'succeeded'
+          },
+          'STATISTICAL_DATA' => {
+            'DATA_INF' => {
+              'VALUE' => data_inf
+            },
+            'CLASS_INF' => {
+              'CLASS_OBJ' => class_obj
+            }
+          }
+        }
+      }
+
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-ok.json'))
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_correct'
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data_default.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@test_data_path)
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('parsing records with default option') do
+      test_data_path = @test_data_path
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'valid')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 2, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with hierarchy_selection') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'parent')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(3, records.length)
+      assert_equal(3, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'child')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(4, records.length)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'both')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(5, records.length)
+      assert_equal(5, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with skip_nil_(column|row)') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_row: true,
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(1, records.length)
+      assert_equal(1 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+    end
+  end
+
+  sub_test_case('anomaly responses') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+      @response_data = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 100,
+            'ERROR_MSG' => 'error message'
+          }
+        }
+      }
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-error.json'))
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('forbidden access with invalid app_id') do
+      test_data_path = @test_data_path
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_invalid'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      cache_file_path = nil
+      stats_data.instance_eval do
+        cache_file_path = @data_path = test_data_path
+      end
+      assert_raise(Datasets::EStatJapan::APIError) do
+        # contains no data
+        stats_data.each do |record|
+          record
+        end
+      end
+      # ensure remove error response cache
+      assert_equal(cache_file_path.exist?, false)
+    end
+  end
+end

data/test/test-fashion-mnist.rb

@@ -0,0 +1,137 @@
+class FashionMNISTTest < Test::Unit::TestCase
+  sub_test_case("Normal") do
+    sub_test_case("train") do
+      def setup
+        @dataset = Datasets::FashionMNIST.new(type: :train)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       60000,
+                       [
+                         9,
+                         784,
+                         [0, 0, 0, 0, 237, 226, 217, 223, 222, 219],
+                         [220, 232, 246, 0, 3, 202, 228, 224, 221, 211],
+                       ],
+                       [
+                         5,
+                         784,
+                         [129, 153, 34, 0, 3, 3, 0, 3, 0, 24],
+                         [180, 177, 177, 47, 101, 235, 194, 223, 232, 255],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [0, 0, 0, 0, 237, 226, 217, 223, 222, 219],
+                       [129, 153, 34, 0, 3, 3, 0, 3, 0, 24],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("fashion-mnist-train", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Fashion-MNIST: train", @dataset.metadata.name)
+        end
+      end
+    end
+
+    sub_test_case("test") do
+      def setup
+        @dataset = Datasets::FashionMNIST.new(type: :test)
+      end
+
+      test("#each") do
+        records = @dataset.each.to_a
+        assert_equal([
+                       10000,
+                       [
+                         9,
+                         784,
+                         [1, 0, 0, 0, 98, 136, 110, 109, 110, 162],
+                         [172, 161, 189, 62, 0, 68, 94, 90, 111, 114],
+                       ],
+                       [
+                         5,
+                         784,
+                         [45, 45, 69, 128, 100, 120, 132, 123, 135, 171],
+                         [63, 74, 72, 0, 1, 0, 0, 0, 4, 85],
+                       ],
+                     ],
+                     [
+                       records.size,
+                       [
+                         records[0].label,
+                         records[0].pixels.size,
+                         records[0].pixels[400, 10],
+                         records[0].pixels[500, 10],
+                       ],
+                       [
+                         records[-1].label,
+                         records[-1].pixels.size,
+                         records[-1].pixels[400, 10],
+                         records[-1].pixels[500, 10],
+                       ],
+                     ])
+      end
+
+      test("#to_table") do
+        table_data = @dataset.to_table
+        assert_equal([
+                       [1, 0, 0, 0, 98, 136, 110, 109, 110, 162],
+                       [45, 45, 69, 128, 100, 120, 132, 123, 135, 171],
+                     ],
+                     [
+                       table_data[:pixels][0][400, 10],
+                       table_data[:pixels][-1][400, 10],
+                     ])
+      end
+
+      sub_test_case("#metadata") do
+        test("#id") do
+          assert_equal("fashion-mnist-test", @dataset.metadata.id)
+        end
+
+        test("#name") do
+          assert_equal("Fashion-MNIST: test", @dataset.metadata.name)
+        end
+      end
+    end
+  end
+
+  sub_test_case("Abnormal") do
+    test("invalid type") do
+      invalid_type = :invalid
+      message = "Please set type :train or :test: #{invalid_type.inspect}"
+      assert_raise(ArgumentError.new(message)) do
+        Datasets::FashionMNIST.new(type: invalid_type)
+      end
+    end
+  end
+end