red-datasets 0.1.0 → 0.1.4

data/test/test-dataset.rb

@@ -0,0 +1,27 @@
+class TestDataset < Test::Unit::TestCase
+  sub_test_case("#clear_cache!") do
+    def setup
+      @dataset = Datasets::Iris.new
+      @cache_dir_path = @dataset.send(:cache_dir_path)
+    end
+
+    test("when the dataset is downloaded") do
+      @dataset.first # This ensures the dataset downloaded
+      existence = {before: @cache_dir_path.join("iris.csv").exist?}
+
+      @dataset.clear_cache!
+      existence[:after] = @cache_dir_path.join("iris.csv").exist?
+
+      assert_equal({before: true, after: false},
+                   existence)
+    end
+
+    test("when the dataset is not downloaded") do
+      FileUtils.rmtree(@cache_dir_path.to_s, secure: true) if @cache_dir_path.exist?
+
+      assert_nothing_raised do
+        @dataset.clear_cache!
+      end
+    end
+  end
+end

data/test/test-downloader.rb

@@ -0,0 +1,29 @@
+class DownloaderTest < Test::Unit::TestCase
+  include Helper::Sandbox
+
+  sub_test_case("#download") do
+    def setup
+      setup_sandbox
+    end
+
+    def teardown
+      teardown_sandbox
+    end
+
+    test("too many redirection") do
+      first_url = "https://example.com/file"
+      last_url = "https://example.com/last_redirection"
+      expected_message = "too many redirections: #{first_url} .. #{last_url}"
+      output_path = @tmp_dir + "file"
+      downloader = Datasets::Downloader.new(first_url)
+
+      downloader.define_singleton_method(:start_http) do |url, headers|
+        raise Datasets::Downloader::TooManyRedirects, "too many redirections: #{last_url}"
+      end
+
+      assert_raise(Datasets::Downloader::TooManyRedirects.new(expected_message)) do
+        downloader.download(output_path)
+      end
+    end
+  end
+end

data/test/test-e-stat-japan.rb

@@ -0,0 +1,383 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'tmpdir'
+
+class EStatJapanTest < Test::Unit::TestCase
+  sub_test_case('app_id') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('nothing') do
+      assert_raise(Datasets::EStatJapan::ArgumentError) do
+        Datasets::EStatJapan::StatsData.new('test-data-id')
+      end
+    end
+
+    test('constructor') do
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+
+    test('env') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_env', stats_data.app_id)
+    end
+
+    test('configure') do
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure & constructor') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+  end
+
+  sub_test_case('url generation') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('generates url correctly') do
+      Datasets::EStatJapan.app_id = 'abcdef'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      stats_data_id = '000000'
+      stats_data.instance_eval do
+        @id = stats_data_id
+        @base_url = 'http://testurl/rest/2.1/app/json/getStatsData'
+      end
+      url = stats_data.send(:generate_url)
+      assert_equal(
+        'http://testurl/rest/2.1/app/json/getStatsData' \
+        '?appId=abcdef&lang=J&statsDataId=000000&' \
+        'metaGetFlg=Y&cntGetFlg=N&sectionHeaderFlg=1',
+        url.to_s
+      )
+    end
+  end
+
+  sub_test_case('parsing records') do
+    def setup
+      Datasets::EStatJapan.app_id = nil
+      # prepare test data
+      class_obj = [
+        {
+          "@name": 'table1',
+          "@id": 'tab',
+          "CLASS": {
+            "@level": '1',
+            "@code": '00001',
+            "@name": 'table1'
+          }
+        },
+        {
+          "@name": 'data1',
+          "@id": 'cat01',
+          "CLASS": {
+            "@level": '1',
+            "@code": 'data1',
+            "@name": 'data1_name'
+          }
+        },
+        {
+          "@name": 'area1',
+          "@id": 'area',
+          "CLASS": [
+            {
+              "@level": '2',
+              "@code": '01100',
+              "@name": 'test1 big-city',
+              "@parentCode": '01000'
+            },
+            {
+              "@level": '3',
+              "@code": '01101',
+              "@name": 'test1 big-city a-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '3',
+              "@code": '01102',
+              "@name": 'test1 big-city b-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '2',
+              "@code": '02555',
+              "@name": 'test2 a-city',
+              "@parentCode": '02000'
+            },
+            {
+              "@level": '2',
+              "@code": '02556',
+              "@name": 'test2 b-city',
+              "@parentCode": '02000'
+            }
+          ]
+        },
+        {
+          "@name": 'time',
+          "@id": 'time',
+          "CLASS": [
+            {
+              "@level": '1',
+              "@code": 'time1',
+              "@name": 'time1'
+            },
+            {
+              "@level": '1',
+              "@code": 'time2',
+              "@name": 'time2'
+            },
+            {
+              "@level": '1',
+              "@code": 'time3',
+              "@name": 'time3'
+            }
+          ]
+        }
+      ]
+      data_inf = class_obj[2][:CLASS].map do |entry|
+        [
+          {
+            "$": 1000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time1',
+            "@unit": 'person'
+          },
+          {
+            "$": 2000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time2',
+            "@unit": 'person'
+          }
+        ]
+      end.flatten
+      ## test record for `skip_nil_row: true`
+      data_inf << {
+        "$": 3000,
+        "@area": '02556',
+        "@cat01": 'data1',
+        "@tab": 'table1',
+        "@time": 'time3',
+        "@unit": 'person'
+      }
+      @response_data_default = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 0,
+            'ERROR_MSG' => 'succeeded'
+          },
+          'STATISTICAL_DATA' => {
+            'DATA_INF' => {
+              'VALUE' => data_inf
+            },
+            'CLASS_INF' => {
+              'CLASS_OBJ' => class_obj
+            }
+          }
+        }
+      }
+
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-ok.json'))
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_correct'
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data_default.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@test_data_path)
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('parsing records with default option') do
+      test_data_path = @test_data_path
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'valid')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 2, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with hierarchy_selection') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'parent')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(3, records.length)
+      assert_equal(3, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'child')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(4, records.length)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'both')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(5, records.length)
+      assert_equal(5, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with skip_nil_(column|row)') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_row: true,
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(1, records.length)
+      assert_equal(1 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+    end
+  end
+
+  sub_test_case('anomaly responses') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+      @response_data = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 100,
+            'ERROR_MSG' => 'error message'
+          }
+        }
+      }
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-error.json'))
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('forbidden access with invalid app_id') do
+      test_data_path = @test_data_path
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_invalid'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      cache_file_path = nil
+      stats_data.instance_eval do
+        cache_file_path = @data_path = test_data_path
+      end
+      assert_raise(Datasets::EStatJapan::APIError) do
+        # contains no data
+        stats_data.each do |record|
+          record
+        end
+      end
+      # ensure remove error response cache
+      assert_equal(cache_file_path.exist?, false)
+    end
+  end
+end

data/test/test-penguins.rb

@@ -0,0 +1,251 @@
+class PenguinsTest < Test::Unit::TestCase
+  sub_test_case("PenguinsRawData::SpeciesBase") do
+    test("#data_path") do
+      data_paths = [ Datasets::PenguinsRawData::Adelie,
+                     Datasets::PenguinsRawData::Gentoo,
+                     Datasets::PenguinsRawData::Chinstrap ].map {|cls|
+                       dataset = cls.new
+                       dataset.data_path.relative_path_from(dataset.send(:cache_dir_path)).to_s
+                     }
+      assert_equal(["penguins/adelie.csv", "penguins/gentoo.csv", "penguins/chinstrap.csv"],
+                   data_paths)
+    end
+  end
+
+  sub_test_case("Adelie") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Adelie.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 152,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Adelie Penguin (Pygoscelis adeliae)",
+                       region: "Anvers",
+                       island: "Torgersen",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N1A1",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2007, 11, 11),
+                       culmen_length_mm: 39.1,
+                       culmen_depth_mm: 18.7,
+                       flipper_length_mm: 181,
+                       body_mass_g: 3750,
+                       sex: "MALE",
+                       delta_15_n_permil: nil,
+                       delta_13_c_permil: nil,
+                       comments: "Not enough blood for isotopes."
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number: 152,
+                       species: "Adelie Penguin (Pygoscelis adeliae)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N85A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 17),
+                       culmen_length_mm: 41.5,
+                       culmen_depth_mm: 18.5,
+                       flipper_length_mm: 201,
+                       body_mass_g: 4000,
+                       sex: "MALE",
+                       delta_15_n_permil: 8.89640,
+                       delta_13_c_permil: -26.06967,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Gentoo") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Gentoo.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 124,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Gentoo penguin (Pygoscelis papua)",
+                       region: "Anvers",
+                       island: "Biscoe",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N31A1",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2007, 11, 27),
+                       culmen_length_mm: 46.1,
+                       culmen_depth_mm: 13.2,
+                       flipper_length_mm: 211,
+                       body_mass_g: 4500,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 7.993,
+                       delta_13_c_permil: -25.5139,
+                       comments: nil
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number:  124,
+                       species: "Gentoo penguin (Pygoscelis papua)",
+                       region: "Anvers",
+                       island: "Biscoe",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N43A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 22),
+                       culmen_length_mm: 49.9,
+                       culmen_depth_mm: 16.1,
+                       flipper_length_mm: 213,
+                       body_mass_g: 5400,
+                       sex: "MALE",
+                       delta_15_n_permil: 8.3639,
+                       delta_13_c_permil: -26.15531,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Chinstrap") do
+    def setup
+      @dataset = Datasets::PenguinsRawData::Chinstrap.new
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([ 68,
+                     {
+                       study_name: "PAL0708",
+                       sample_number: 1,
+                       species: "Chinstrap penguin (Pygoscelis antarctica)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N61A1",
+                       clutch_completion: "No",
+                       date_egg: DateTime.new(2007, 11, 19),
+                       culmen_length_mm: 46.5,
+                       culmen_depth_mm: 17.9,
+                       flipper_length_mm: 192,
+                       body_mass_g: 3500,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 9.03935,
+                       delta_13_c_permil: -24.30229,
+                       comments: "Nest never observed with full clutch."
+                     },
+                     {
+                       study_name: "PAL0910",
+                       sample_number: 68,
+                       species: "Chinstrap penguin (Pygoscelis antarctica)",
+                       region: "Anvers",
+                       island: "Dream",
+                       stage: "Adult, 1 Egg Stage",
+                       individual_id: "N100A2",
+                       clutch_completion: "Yes",
+                       date_egg: DateTime.new(2009, 11, 21),
+                       culmen_length_mm: 50.2,
+                       culmen_depth_mm: 18.7,
+                       flipper_length_mm: 198,
+                       body_mass_g: 3775,
+                       sex: "FEMALE",
+                       delta_15_n_permil: 9.39305,
+                       delta_13_c_permil: -24.25255,
+                       comments: nil
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("Penguins") do
+    def setup
+      @dataset = Datasets::Penguins.new
+    end
+
+    test("order of species") do
+      species_values = @dataset.map {|r| r.species }.uniq
+      assert_equal(["Adelie", "Chinstrap", "Gentoo"],
+                   species_values)
+    end
+
+    test("data cleansing") do
+      sex_values = @dataset.map {|r| r.sex }.uniq.compact.sort
+      assert_equal(["female", "male"],
+                   sex_values)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     344,
+                     {
+                       species: "Adelie",
+                       island: "Torgersen",
+                       bill_length_mm: 39.1,
+                       bill_depth_mm: 18.7,
+                       flipper_length_mm: 181,
+                       body_mass_g: 3750,
+                       sex: "male",
+                       year: 2007
+                     },
+                     {
+                       species: "Chinstrap",
+                       island: "Dream",
+                       bill_length_mm: 46.5,
+                       bill_depth_mm: 17.9,
+                       flipper_length_mm: 192,
+                       body_mass_g: 3500,
+                       sex: "female",
+                       year: 2007
+                     },
+                     {
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 46.1,
+                       bill_depth_mm: 13.2,
+                       flipper_length_mm: 211,
+                       body_mass_g: 4500,
+                       sex: "female",
+                       year: 2007
+                     },
+                     {
+                       species: "Gentoo",
+                       island: "Biscoe",
+                       bill_length_mm: 49.9,
+                       bill_depth_mm: 16.1,
+                       flipper_length_mm: 213,
+                       body_mass_g: 5400,
+                       sex: "male",
+                       year: 2009
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[152].to_h,
+                     records[220].to_h,
+                     records[-1].to_h,
+                   ])
+    end
+  end
+end