red-datasets 0.0.7 → 0.1.2

data/test/test-communities.rb

@@ -0,0 +1,290 @@
+class CommunitiesTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::Communities.new
+  end
+
+  def record(*args)
+    Datasets::Communities::Record.new(*args)
+  end
+
+  test('#each') do
+    records = @dataset.each.to_a
+    assert_equal([
+                    1994,
+                    {
+                      :state => 8,
+                      :county => nil,
+                      :community => nil,
+                      :community_name => "Lakewoodcity",
+                      :fold => 1,
+                      :population => 0.19,
+                      :household_size => 0.33,
+                      :race_percent_black => 0.02,
+                      :race_percent_white => 0.9,
+                      :race_percent_asian => 0.12,
+                      :race_percent_hispanic => 0.17,
+                      :age_percent_12_to_21 => 0.34,
+                      :age_percent_12_to_29 => 0.47,
+                      :age_percent_16_to_24 => 0.29,
+                      :age_percent_65_and_upper => 0.32,
+                      :n_people_urban => 0.2,
+                      :percent_people_urban => 1,
+                      :median_income => 0.37,
+                      :percent_households_with_wage => 0.72,
+                      :percent_households_with_farm_self => 0.34,
+                      :percent_households_with_investment_income => 0.6,
+                      :percent_households_with_social_security => 0.29,
+                      :percent_households_with_public_assistant => 0.15,
+                      :percent_households_with_retire => 0.43,
+                      :median_family_income => 0.39,
+                      :per_capita_income => 0.4,
+                      :per_capita_income_white => 0.39,
+                      :per_capita_income_black => 0.32,
+                      :per_capita_income_indian => 0.27,
+                      :per_capita_income_asian => 0.27,
+                      :per_capita_income_other => 0.36,
+                      :per_capita_income_hispanic => 0.41,
+                      :n_people_under_poverty => 0.08,
+                      :percent_people_under_poverty => 0.19,
+                      :percent_less_9th_grade => 0.1,
+                      :percent_not_high_school_graduate => 0.18,
+                      :percent_bachelors_or_more => 0.48,
+                      :percent_unemployed => 0.27,
+                      :percent_employed => 0.68,
+                      :percent_employed_manufacturing => 0.23,
+                      :percent_employed_professional_service => 0.41,
+                      :percent_occupations_manufacturing => 0.25,
+                      :percent_occupations_management_professional => 0.52,
+                      :male_percent_divorced => 0.68,
+                      :male_percent_never_married => 0.4,
+                      :female_percent_divorced => 0.75,
+                      :total_percent_divorced => 0.75,
+                      :mean_persons_per_family => 0.35,
+                      :percent_family_2_parents => 0.55,
+                      :percent_kids_2_parents => 0.59,
+                      :percent_young_kids_2_parents => 0.61,
+                      :percent_teen_2_parents => 0.56,
+                      :percent_work_mom_young_kids => 0.74,
+                      :percent_work_mom => 0.76,
+                      :n_illegals => 0.04,
+                      :percent_illegals => 0.14,
+                      :n_immigrants => 0.03,
+                      :percent_immigrants_recent => 0.24,
+                      :percent_immigrants_recent_5 => 0.27,
+                      :percent_immigrants_recent_8 => 0.37,
+                      :percent_immigrants_recent_10 => 0.39,
+                      :percent_population_immigranted_recent => 0.07,
+                      :percent_population_immigranted_recent_5 => 0.07,
+                      :percent_population_immigranted_recent_8 => 0.08,
+                      :percent_population_immigranted_recent_10 => 0.08,
+                      :percent_speak_english_only => 0.89,
+                      :percent_not_speak_english_well => 0.06,
+                      :percent_large_households_family => 0.14,
+                      :percent_large_households_occupied => 0.13,
+                      :mean_persons_per_occupied_household => 0.33,
+                      :mean_persons_per_owner_occupied_household => 0.39,
+                      :mean_persons_per_rental_occupied_household => 0.28,
+                      :percent_persons_owner_occupied_household => 0.55,
+                      :percent_persons_dense_housing => 0.09,
+                      :percent_housing_less_3_bedrooms => 0.51,
+                      :median_n_bedrooms => 0.5,
+                      :n_vacant_households => 0.21,
+                      :percent_housing_occupied => 0.71,
+                      :percent_housing_owner_occupied => 0.52,
+                      :percent_vacant_housing_boarded => 0.05,
+                      :percent_vacant_housing_more_6_months => 0.26,
+                      :median_year_housing_built => 0.65,
+                      :percent_housing_no_phone => 0.14,
+                      :percent_housing_without_full_plumbing => 0.06,
+                      :owner_occupied_housing_lower_quartile => 0.22,
+                      :owner_occupied_housing_median => 0.19,
+                      :owner_occupied_housing_higher_quartile => 0.18,
+                      :rental_housing_lower_quartile => 0.36,
+                      :rental_housing_median => 0.35,
+                      :rental_housing_higher_quartile => 0.38,
+                      :median_rent => 0.34,
+                      :median_rent_percent_household_income => 0.38,
+                      :median_owner_cost_percent_household_income => 0.46,
+                      :median_owner_cost_percent_household_income_no_mortgage => 0.25,
+                      :n_people_shelter => 0.04,
+                      :n_people_street => 0,
+                      :percent_foreign_born => 0.12,
+                      :percent_born_same_state => 0.42,
+                      :percent_same_house_85 => 0.5,
+                      :percent_same_city_85 => 0.51,
+                      :percent_same_state_85 => 0.64,
+                      :lemas_sworn_full_time => 0.03,
+                      :lemas_sworn_full_time_per_population => 0.13,
+                      :lemas_sworn_full_time_field => 0.96,
+                      :lemas_sworn_full_time_field_per_population => 0.17,
+                      :lemas_total_requests => 0.06,
+                      :lemas_total_requests_per_population => 0.18,
+                      :total_requests_per_officer => 0.44,
+                      :n_officers_per_population => 0.13,
+                      :racial_match_community_police => 0.94,
+                      :percent_police_white => 0.93,
+                      :percent_police_black => 0.03,
+                      :percent_police_hispanic => 0.07,
+                      :percent_police_asian => 0.1,
+                      :percent_police_minority => 0.07,
+                      :n_officers_assigned_drug_units => 0.02,
+                      :n_kinds_drugs_seized => 0.57,
+                      :police_average_overtime_worked => 0.29,
+                      :land_area => 0.12,
+                      :population_density => 0.26,
+                      :percent_use_public_transit => 0.2,
+                      :n_police_cars => 0.06,
+                      :n_police_operating_budget => 0.04,
+                      :lemas_percent_police_on_patrol => 0.9,
+                      :lemas_gang_unit_deployed => 0.5,
+                      :lemas_percent_office_drug_units => 0.32,
+                      :police_operating_budget_per_population => 0.14,
+                      :total_violent_crimes_per_population => 0.2,
+                    },
+                    {
+                      :state => 6,
+                      :county => nil,
+                      :community => nil,
+                      :community_name => "Ontariocity",
+                      :fold => 10,
+                      :population => 0.2,
+                      :household_size => 0.78,
+                      :race_percent_black => 0.14,
+                      :race_percent_white => 0.46,
+                      :race_percent_asian => 0.24,
+                      :race_percent_hispanic => 0.77,
+                      :age_percent_12_to_21 => 0.5,
+                      :age_percent_12_to_29 => 0.62,
+                      :age_percent_16_to_24 => 0.4,
+                      :age_percent_65_and_upper => 0.17,
+                      :n_people_urban => 0.21,
+                      :percent_people_urban => 1,
+                      :median_income => 0.4,
+                      :percent_households_with_wage => 0.73,
+                      :percent_households_with_farm_self => 0.22,
+                      :percent_households_with_investment_income => 0.25,
+                      :percent_households_with_social_security => 0.26,
+                      :percent_households_with_public_assistant => 0.47,
+                      :percent_households_with_retire => 0.29,
+                      :median_family_income => 0.36,
+                      :per_capita_income => 0.24,
+                      :per_capita_income_white => 0.28,
+                      :per_capita_income_black => 0.32,
+                      :per_capita_income_indian => 0.22,
+                      :per_capita_income_asian => 0.27,
+                      :per_capita_income_other => 0.25,
+                      :per_capita_income_hispanic => 0.29,
+                      :n_people_under_poverty => 0.16,
+                      :percent_people_under_poverty => 0.35,
+                      :percent_less_9th_grade => 0.5,
+                      :percent_not_high_school_graduate => 0.55,
+                      :percent_bachelors_or_more => 0.16,
+                      :percent_unemployed => 0.47,
+                      :percent_employed => 0.58,
+                      :percent_employed_manufacturing => 0.53,
+                      :percent_employed_professional_service => 0.2,
+                      :percent_occupations_manufacturing => 0.6,
+                      :percent_occupations_management_professional => 0.24,
+                      :male_percent_divorced => 0.49,
+                      :male_percent_never_married => 0.5,
+                      :female_percent_divorced => 0.6,
+                      :total_percent_divorced => 0.57,
+                      :mean_persons_per_family => 0.86,
+                      :percent_family_2_parents => 0.61,
+                      :percent_kids_2_parents => 0.59,
+                      :percent_young_kids_2_parents => 0.64,
+                      :percent_teen_2_parents => 0.6,
+                      :percent_work_mom_young_kids => 0.35,
+                      :percent_work_mom => 0.35,
+                      :n_illegals => 0.11,
+                      :percent_illegals => 0.43,
+                      :n_immigrants => 0.2,
+                      :percent_immigrants_recent => 0.43,
+                      :percent_immigrants_recent_5 => 0.52,
+                      :percent_immigrants_recent_8 => 0.58,
+                      :percent_immigrants_recent_10 => 0.65,
+                      :percent_population_immigranted_recent => 0.68,
+                      :percent_population_immigranted_recent_5 => 0.73,
+                      :percent_population_immigranted_recent_8 => 0.73,
+                      :percent_population_immigranted_recent_10 => 0.75,
+                      :percent_speak_english_only => 0.35,
+                      :percent_not_speak_english_well => 0.77,
+                      :percent_large_households_family => 0.84,
+                      :percent_large_households_occupied => 0.83,
+                      :mean_persons_per_occupied_household => 0.84,
+                      :mean_persons_per_owner_occupied_household => 0.78,
+                      :mean_persons_per_rental_occupied_household => 0.89,
+                      :percent_persons_owner_occupied_household => 0.46,
+                      :percent_persons_dense_housing => 0.76,
+                      :percent_housing_less_3_bedrooms => 0.55,
+                      :median_n_bedrooms => 0,
+                      :n_vacant_households => 0.12,
+                      :percent_housing_occupied => 0.8,
+                      :percent_housing_owner_occupied => 0.49,
+                      :percent_vacant_housing_boarded => 0.1,
+                      :percent_vacant_housing_more_6_months => 0.16,
+                      :median_year_housing_built => 0.71,
+                      :percent_housing_no_phone => 0.32,
+                      :percent_housing_without_full_plumbing => 0.21,
+                      :owner_occupied_housing_lower_quartile => 0.35,
+                      :owner_occupied_housing_median => 0.35,
+                      :owner_occupied_housing_higher_quartile => 0.32,
+                      :rental_housing_lower_quartile => 0.54,
+                      :rental_housing_median => 0.53,
+                      :rental_housing_higher_quartile => 0.61,
+                      :median_rent => 0.54,
+                      :median_rent_percent_household_income => 0.69,
+                      :median_owner_cost_percent_household_income => 0.73,
+                      :median_owner_cost_percent_household_income_no_mortgage => 0.21,
+                      :n_people_shelter => 0.08,
+                      :n_people_street => 0.08,
+                      :percent_foreign_born => 0.68,
+                      :percent_born_same_state => 0.5,
+                      :percent_same_house_85 => 0.34,
+                      :percent_same_city_85 => 0.35,
+                      :percent_same_state_85 => 0.68,
+                      :lemas_sworn_full_time => 0.03,
+                      :lemas_sworn_full_time_per_population => 0.11,
+                      :lemas_sworn_full_time_field => 0.96,
+                      :lemas_sworn_full_time_field_per_population => 0.14,
+                      :lemas_total_requests => 0.04,
+                      :lemas_total_requests_per_population => 0.11,
+                      :total_requests_per_officer => 0.3,
+                      :n_officers_per_population => 0.11,
+                      :racial_match_community_police => 0.45,
+                      :percent_police_white => 0.74,
+                      :percent_police_black => 0.34,
+                      :percent_police_hispanic => 0.07,
+                      :percent_police_asian => 0,
+                      :percent_police_minority => 0.28,
+                      :n_officers_assigned_drug_units => 0.07,
+                      :n_kinds_drugs_seized => 0.36,
+                      :police_average_overtime_worked => 0.11,
+                      :land_area => 0.11,
+                      :population_density => 0.3,
+                      :percent_use_public_transit => 0.05,
+                      :n_police_cars => 0.08,
+                      :n_police_operating_budget => 0.04,
+                      :lemas_percent_police_on_patrol => 0.73,
+                      :lemas_gang_unit_deployed => 0.5,
+                      :lemas_percent_office_drug_units => 1,
+                      :police_operating_budget_per_population => 0.13,
+                      :total_violent_crimes_per_population => 0.48
+                    },
+                 ],
+                 [
+                    records.size,
+                    records[0].to_h,
+                    records[-1].to_h
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("Title: Communities and Crime")
+      end
+    end
+  end
+end

data/test/test-dataset.rb

@@ -0,0 +1,27 @@
+class TestDataset < Test::Unit::TestCase
+  sub_test_case("#clear_cache!") do
+    def setup
+      @dataset = Datasets::Iris.new
+      @cache_dir_path = @dataset.send(:cache_dir_path)
+    end
+
+    test("when the dataset is downloaded") do
+      @dataset.first # This ensures the dataset downloaded
+      existence = {before: @cache_dir_path.join("iris.csv").exist?}
+
+      @dataset.clear_cache!
+      existence[:after] = @cache_dir_path.join("iris.csv").exist?
+
+      assert_equal({before: true, after: false},
+                   existence)
+    end
+
+    test("when the dataset is not downloaded") do
+      FileUtils.rmtree(@cache_dir_path.to_s, secure: true) if @cache_dir_path.exist?
+
+      assert_nothing_raised do
+        @dataset.clear_cache!
+      end
+    end
+  end
+end

data/test/test-downloader.rb

@@ -0,0 +1,29 @@
+class DownloaderTest < Test::Unit::TestCase
+  include Helper::Sandbox
+
+  sub_test_case("#download") do
+    def setup
+      setup_sandbox
+    end
+
+    def teardown
+      teardown_sandbox
+    end
+
+    test("too many redirection") do
+      first_url = "https://example.com/file"
+      last_url = "https://example.com/last_redirection"
+      expected_message = "too many redirections: #{first_url} .. #{last_url}"
+      output_path = @tmp_dir + "file"
+      downloader = Datasets::Downloader.new(first_url)
+
+      downloader.define_singleton_method(:start_http) do |url, headers|
+        raise Datasets::Downloader::TooManyRedirects, "too many redirections: #{last_url}"
+      end
+
+      assert_raise(Datasets::Downloader::TooManyRedirects.new(expected_message)) do
+        downloader.download(output_path)
+      end
+    end
+  end
+end

data/test/test-e-stat-japan.rb

@@ -0,0 +1,383 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'tmpdir'
+
+class EStatJapanTest < Test::Unit::TestCase
+  sub_test_case('app_id') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('nothing') do
+      assert_raise(Datasets::EStatJapan::ArgumentError) do
+        Datasets::EStatJapan::StatsData.new('test-data-id')
+      end
+    end
+
+    test('constructor') do
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+
+    test('env') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_env', stats_data.app_id)
+    end
+
+    test('configure') do
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      assert_equal('test_by_configure', stats_data.app_id)
+    end
+
+    test('env & configure & constructor') do
+      ENV['ESTATJAPAN_APP_ID'] = 'test_by_env'
+      Datasets::EStatJapan.configure do |config|
+        config.app_id = 'test_by_configure'
+      end
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'test_by_constructor')
+      assert_equal('test_by_constructor', stats_data.app_id)
+    end
+  end
+
+  sub_test_case('url generation') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+    end
+
+    test('generates url correctly') do
+      Datasets::EStatJapan.app_id = 'abcdef'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      stats_data_id = '000000'
+      stats_data.instance_eval do
+        @id = stats_data_id
+        @base_url = 'http://testurl/rest/2.1/app/json/getStatsData'
+      end
+      url = stats_data.send(:generate_url)
+      assert_equal(
+        'http://testurl/rest/2.1/app/json/getStatsData' \
+        '?appId=abcdef&lang=J&statsDataId=000000&' \
+        'metaGetFlg=Y&cntGetFlg=N&sectionHeaderFlg=1',
+        url.to_s
+      )
+    end
+  end
+
+  sub_test_case('parsing records') do
+    def setup
+      Datasets::EStatJapan.app_id = nil
+      # prepare test data
+      class_obj = [
+        {
+          "@name": 'table1',
+          "@id": 'tab',
+          "CLASS": {
+            "@level": '1',
+            "@code": '00001',
+            "@name": 'table1'
+          }
+        },
+        {
+          "@name": 'data1',
+          "@id": 'cat01',
+          "CLASS": {
+            "@level": '1',
+            "@code": 'data1',
+            "@name": 'data1_name'
+          }
+        },
+        {
+          "@name": 'area1',
+          "@id": 'area',
+          "CLASS": [
+            {
+              "@level": '2',
+              "@code": '01100',
+              "@name": 'test1 big-city',
+              "@parentCode": '01000'
+            },
+            {
+              "@level": '3',
+              "@code": '01101',
+              "@name": 'test1 big-city a-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '3',
+              "@code": '01102',
+              "@name": 'test1 big-city b-ku',
+              "@parentCode": '01100'
+            },
+            {
+              "@level": '2',
+              "@code": '02555',
+              "@name": 'test2 a-city',
+              "@parentCode": '02000'
+            },
+            {
+              "@level": '2',
+              "@code": '02556',
+              "@name": 'test2 b-city',
+              "@parentCode": '02000'
+            }
+          ]
+        },
+        {
+          "@name": 'time',
+          "@id": 'time',
+          "CLASS": [
+            {
+              "@level": '1',
+              "@code": 'time1',
+              "@name": 'time1'
+            },
+            {
+              "@level": '1',
+              "@code": 'time2',
+              "@name": 'time2'
+            },
+            {
+              "@level": '1',
+              "@code": 'time3',
+              "@name": 'time3'
+            }
+          ]
+        }
+      ]
+      data_inf = class_obj[2][:CLASS].map do |entry|
+        [
+          {
+            "$": 1000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time1',
+            "@unit": 'person'
+          },
+          {
+            "$": 2000,
+            "@area": entry[:@code],
+            "@cat01": 'data1',
+            "@tab": 'table1',
+            "@time": 'time2',
+            "@unit": 'person'
+          }
+        ]
+      end.flatten
+      ## test record for `skip_nil_row: true`
+      data_inf << {
+        "$": 3000,
+        "@area": '02556',
+        "@cat01": 'data1',
+        "@tab": 'table1',
+        "@time": 'time3',
+        "@unit": 'person'
+      }
+      @response_data_default = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 0,
+            'ERROR_MSG' => 'succeeded'
+          },
+          'STATISTICAL_DATA' => {
+            'DATA_INF' => {
+              'VALUE' => data_inf
+            },
+            'CLASS_INF' => {
+              'CLASS_OBJ' => class_obj
+            }
+          }
+        }
+      }
+
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-ok.json'))
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_correct'
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data_default.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@test_data_path)
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('parsing records with default option') do
+      test_data_path = @test_data_path
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id', app_id: 'valid')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 2, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with hierarchy_selection') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'parent')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(3, records.length)
+      assert_equal(3, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'child')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(4, records.length)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            hierarchy_selection: 'both')
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      stats_data.each do |record|
+        records << record
+      end
+      assert_equal(5, records.length)
+      assert_equal(5, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(2, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(2, stats_data.schema.length)
+    end
+
+    test('parsing records with skip_nil_(column|row)') do
+      test_data_path = @test_data_path
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(4, records.length)
+      assert_equal(4 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+
+      stats_data = \
+        Datasets::EStatJapan::StatsData.new('test-data-id',
+                                            skip_nil_row: true,
+                                            skip_nil_column: false)
+      stats_data.instance_eval do
+        @data_path = test_data_path
+      end
+      records = []
+      value_num = 0
+      stats_data.each do |record|
+        records << record
+        value_num += record.values.length
+      end
+      assert_equal(1, records.length)
+      assert_equal(1 * 3, value_num)
+      assert_equal(4, stats_data.areas.length)
+      assert_equal(3, stats_data.time_tables.length)
+      assert_equal(3, stats_data.time_tables.reject { |_k, v| v[:skip] }.to_h.length)
+      assert_equal(1, stats_data.columns.length)
+      assert_equal(3, stats_data.schema.length)
+    end
+  end
+
+  sub_test_case('anomaly responses') do
+    def setup
+      ENV['ESTATJAPAN_APP_ID'] = nil
+      Datasets::EStatJapan.app_id = nil
+      @response_data = {
+        'GET_STATS_DATA' => {
+          'RESULT' => {
+            'STATUS' => 100,
+            'ERROR_MSG' => 'error message'
+          }
+        }
+      }
+      @tmp_dir = Dir.mktmpdir
+      @test_data_path = Pathname(File.join(@tmp_dir, '200-error.json'))
+      File.open(@test_data_path, 'w') do |f|
+        f.write(@response_data.to_json)
+      end
+    end
+
+    def teardown
+      FileUtils.remove_entry_secure(@tmp_dir)
+    end
+
+    test('forbidden access with invalid app_id') do
+      test_data_path = @test_data_path
+      ENV['ESTATJAPAN_APP_ID'] = 'test_appid_invalid'
+      stats_data = Datasets::EStatJapan::StatsData.new('test-data-id')
+      cache_file_path = nil
+      stats_data.instance_eval do
+        cache_file_path = @data_path = test_data_path
+      end
+      assert_raise(Datasets::EStatJapan::APIError) do
+        # contains no data
+        stats_data.each do |record|
+          record
+        end
+      end
+      # ensure remove error response cache
+      assert_equal(cache_file_path.exist?, false)
+    end
+  end
+end