countries 3.0.0 → 4.0.1

data/lib/countries/data/translation_corrections.yaml

@@ -1,6 +1,6 @@
 # Contains fixes for upstream pkg-iso-codes
 # Any change here should have an upstream bug report
-# https://alioth.debian.org/projects/pkg-iso-codes/
+# https://salsa.debian.org/iso-codes-team/iso-codes
 #
 # These changes are applied during `rake update_cache`
 #
@@ -21,8 +21,12 @@ ar:
   SY: "سوريا"
   BQ: "بونير، سينت أوستاتيوس وسابا"
   CW: "كوراساو"
+nn:
+  PT: Portugal
 zh:
   KH: '柬埔寨'
+zh_cn:
+  KH: '柬埔寨'
 fa:
   IR: "ایران"
 he:
@@ -30,3 +34,8 @@ he:
   VI: "איי הבתולה (ארה)"
 sv:
   GB: "Storbritannien"
+  RU: "Ryssland"
+pt:
+  CZ: 'República Checa'
+da:
+  CZ: 'Tjekkiet'

data/lib/countries/sources/cldr/downloader.rb

@@ -2,6 +2,8 @@ require 'uri'
 require 'net/http'
 require 'nokogiri'
 require 'fileutils'
+require 'json'
+
 module Sources
   module CLDR
     module Downloader
@@ -14,22 +16,19 @@ module Sources
       def download_folder(type)
         folder = File.join(ISO3166_ROOT_PATH, 'tmp', 'cldr', 'trunk', 'common', type)
         FileUtils.mkdir_p(folder)
-        doc = Nokogiri::HTML get(type + '/')
-        doc.css('a[href]').map { |e| e.attributes['href'].value }.each do |href|
-          next if href == '../'
-          File.write(File.join(folder, href), get([type, href].join('/')))
+        url = URI.parse("https://api.github.com/repos/unicode-org/cldr/contents/common/" + type)
+        path_listing = JSON.parse(Net::HTTP.get_response(url).body)
+        path_listing.each do |path|
+          if path['name'] =~ /\.xml$/
+            File.open(File.join(folder, path['name']), 'w') do |f|
+              raw_url = URI.parse(path['download_url'])
+              f.write(Net::HTTP.get_response(raw_url).body)
+            end
+          end
         end
       end
 
-      def get(path)
-        url = URI("http://www.unicode.org/repos/cldr/trunk/common/#{path}")
-
-        http = Net::HTTP.new(url.host, url.port)
-
-        request = Net::HTTP::Get.new(url)
-        response = http.request(request)
-        response.read_body
-      end
     end
   end
 end
+

data/lib/countries/structure.rb

@@ -42,6 +42,7 @@ module ISO3166
     'languages_spoken' => [],
     'translations' => {},
     'postal_code' => nil,
+    'postal_code_format' => nil,
     'region' => nil,
     'unofficial_names' => [],
     'start_of_week' => 'monday',

data/lib/countries/tasks/geocoding.rake

@@ -2,7 +2,9 @@ require 'geocoder'
 require 'retryable'
 
 Geocoder.configure(
-  timeout: 10
+  lookup: :google,
+  timeout: 10,
+  api_key: GOOGLE_API_KEY
 )
 
 # raise on geocoding errors such as query limit exceeded
@@ -10,9 +12,9 @@ Geocoder.configure(always_raise: :all)
 # Try to geocode a given query, on exceptions it retries up to 3 times then gives up.
 # @param [String] query string to geocode
 # @return [Hash] first valid result or nil
-def geocode(query)
+def geocode(query, params)
   Retryable.retryable(tries: 3, sleep: ->(n) { 2**n }) do
-    Geocoder.search(query).first
+    Geocoder.search(query, params: params).first
   end
 rescue => e
   warn "Attempts exceeded for query #{query}, last error was #{e.message}"
@@ -41,12 +43,8 @@ namespace :geocode do
       # Load unmutated yaml file.
       data = load_country_yaml(country.alpha2)
 
-      lookup = "#{country.alpha2} country"
-      # LU country lookup appears to match to Los Angeles
-      lookup = country.name if country.alpha2 == 'LU'
-
-      next unless (result = geocode(lookup))
-      puts 'WARNING:: Geocoder returned something that was not a country' unless result.types.include?('country')
+      next unless (result = geocode(country.name, {region: country.alpha2}))
+      puts "WARNING:: Geocoder returned something that was not a country for #{country.alpha2}" unless result.types.include?('country')
       geometry = result.geometry
 
       # Extract center point data
@@ -76,15 +74,10 @@ namespace :geocode do
       # Iterate subdivisions
       state_data = c.subdivisions.dup
       state_data.reject { |_, data| data['geo'] }.each do |code, data|
-        location = "#{data['name']}, #{c.name}"
+        location = "#{c.alpha2}-#{code}"
 
-        # Handle special geocoding cases where Google defaults to well known
-        # cities, instead of the states.
-        if c.alpha2 == 'US' && %w(NY WA OK).include?(code)
-          location = "#{data['name']} State, United States"
-        end
+        next unless (result = geocode(location, {region: c.alpha2}))
 
-        next unless (result = geocode(location))
         geometry = result.geometry
         if geometry['location']
           state_data[code]['geo'] ||= {}

data/lib/countries/tasks/postal_codes.rake

@@ -0,0 +1,48 @@
+namespace :postal_codes do
+  desc 'Retrieve and update postal codes and their format'
+  task :update do
+    require 'json'
+    require 'open-uri'
+    require 'yaml'
+
+    country_files = Dir['lib/countries/data/countries/*.yaml'].sort
+
+    country_files.each do |country_file|
+      yaml = YAML.load_file(country_file)
+      country_code = File.basename(country_file, File.extname(country_file))
+      print "\rUpdating #{country_code}"
+
+      data = yaml[country_code].to_a
+      postal_code_index = data.find_index { |d| d[0] == 'postal_code' }
+      postal_code_format_index = data.find_index { |d| d[0] == 'postal_code_format' }
+
+      response = URI.open("https://chromium-i18n.appspot.com/ssl-address/data/#{country_code}").read
+      json = JSON.parse(response) rescue {}
+      puts ' - Returned empty data. Skipping ' and next if json.empty?
+
+      postal_code = ['postal_code', !!json['zip']]
+      postal_code_format = ['postal_code_format', json['zip']]
+
+      if postal_code_index
+        data[postal_code_index] = postal_code
+      else
+        postal_code_index = data.find_index { |d| d[0] == 'nationality' } + 1 || data.size
+        data.insert(postal_code_index, postal_code)
+      end
+
+      if json['zip']
+        if postal_code_format_index
+          data[postal_code_format_index] = postal_code_format
+        else
+          data.insert(postal_code_index + 1, postal_code_format)
+        end
+      elsif postal_code_format_index
+        data.delete_at(postal_code_format_index)
+      end
+
+      yaml[country_code] = data.to_h
+
+      File.open(country_file, 'w') { |file| file.write(yaml.to_yaml) }
+    end
+  end
+end

data/lib/countries/translations.rb

@@ -6,7 +6,7 @@ module ISO3166
   # to `pt` to prevent from showing nil values
   class Translations < Hash
     def [](locale)
-      super(locale) || super(locale.sub(/-.*/, ''))
+      super(locale) || super(locale.to_s.sub(/-.*/, ''))
     end
   end
 end

data/lib/countries/version.rb

@@ -1,3 +1,3 @@
 module Countries
-  VERSION = '3.0.0'.freeze
+  VERSION = '4.0.1'.freeze
 end

data/spec/country_spec.rb

@@ -48,7 +48,7 @@ describe ISO3166::Country do
   end
 
   it 'should return alternate names' do
-    expect(country.names).to eq(['United States', 'Vereinigte Staaten von Amerika', 'États-Unis', 'Estados Unidos', 'アメリカ合衆国', 'Verenigde Staten'])
+    expect(country.names).to eq(['United States', 'Murica', 'Vereinigte Staaten von Amerika', 'États-Unis', 'Estados Unidos', 'アメリカ合衆国', 'Verenigde Staten'])
   end
 
   it 'should return translations' do
@@ -85,8 +85,18 @@ describe ISO3166::Country do
   end
 
   it 'knows when a country does not require postal codes' do
-    ireland = ISO3166::Country.search('IE')
-    expect(ireland.postal_code).to eq(false)
+    antarctica = ISO3166::Country.search('AQ')
+    expect(antarctica.postal_code).to eq(false)
+  end
+
+  it 'knows about the country postal code format' do
+    expect(country.postal_code_format).to_not be_nil
+
+    regex = Regexp.new(country.postal_code_format)
+    expect(regex).to match("12345-6789")
+
+    antarctica = ISO3166::Country.search('AQ')
+    expect(antarctica.postal_code_format).to be_nil
   end
 
   it 'should return region' do
@@ -322,7 +332,7 @@ describe ISO3166::Country do
       countries = ISO3166::Country.all_translated('fr')
       expect(countries).to be_an(Array)
       expect(countries.first).to be_a(String)
-      expect(countries.first).to eq('Afghanistan')
+      expect(countries.first).to eq('Aruba')
       # countries missing the desired locale will not be added to the list
       # so all 250 countries may not be returned, 'fr' returns 249, for example
       expect(countries.size).to eq(NUM_OF_COUNTRIES)
@@ -332,9 +342,29 @@ describe ISO3166::Country do
       countries = ISO3166::Country.all_translated
       expect(countries).to be_an(Array)
       expect(countries.first).to be_a(String)
-      expect(countries.first).to eq('Afghanistan')
+      expect(countries.first).to eq('Aruba')
       expect(countries.size).to eq(NUM_OF_COUNTRIES)
     end
+
+    context 'with custom countries' do
+      before do
+        ISO3166::Data.register(
+          alpha2: 'XX',
+          name: 'Custom Country',
+          translations: { 'en' => 'Custom Country' }
+        )
+      end
+
+      it 'should include custom registered countries' do
+        custom_country = ISO3166::Country.find_by_alpha2('XX')[1]
+        countries = ISO3166::Country.all_translated
+        expect(countries).to include(custom_country['name'])
+      end
+
+      after do
+        ISO3166::Data.unregister('XX')
+      end
+    end
   end
 
   describe 'all_names_with_codes' do
@@ -397,14 +427,31 @@ describe ISO3166::Country do
       countries = ISO3166::Country.new(:de).translation('xxx')
       expect(countries).to be_nil
     end
+
+    context 'should return variant locales' do
+      it 'should return different value for Chinese variants' do
+        ISO3166.configuration.locales = [:'zh-cn', :'zh-hk', :'zh-tw']
+        name_cn = ISO3166::Country['TW'].translation('zh-cn')
+        name_hk = ISO3166::Country['TW'].translation('zh-hk')
+        name_tw = ISO3166::Country['TW'].translation('zh-tw')
+        expect([name_cn, name_hk, name_tw].uniq.size).to eql 3
+      end
+
+      it 'should return different value for Portuguese variants' do
+        ISO3166.configuration.locales = [:pt, :'pt-br']
+        name_pt = ISO3166::Country['SG'].translation('pt')
+        name_br = ISO3166::Country['SG'].translation('pt-br')
+        expect([name_pt, name_br].uniq.size).to eql 2
+      end
+    end
   end
 
   describe 'translations' do
     it 'should return an hash of all country names translated to the selected locale' do
       countries = ISO3166::Country.translations('fr')
       expect(countries).to be_an(Hash)
-      expect(countries.first[0]).to eq('AF')
-      expect(countries.first).to eq(%w(AF Afghanistan))
+      expect(countries.first[0]).to eq('AW')
+      expect(countries.first).to eq(%w(AW Aruba))
       # countries missing the desired locale will not be added to the list
       # so all 250 countries may not be returned, 'fr' returns 249, for example
       expect(countries.size).to eq(NUM_OF_COUNTRIES)
@@ -413,8 +460,8 @@ describe ISO3166::Country do
     it 'should return an hash of all country names in English if no locale is passed' do
       countries = ISO3166::Country.translations
       expect(countries).to be_an(Hash)
-      expect(countries.first[0]).to eq('AF')
-      expect(countries.first).to eq(%w(AF Afghanistan))
+      expect(countries.first[0]).to eq('AW')
+      expect(countries.first).to eq(%w(AW Aruba))
       expect(countries.size).to eq(NUM_OF_COUNTRIES)
     end
   end
@@ -502,6 +549,19 @@ describe ISO3166::Country do
       expect(spain_data).to be_a Hash
       expect(spain_data.keys).to eq(['ES'])
     end
+
+    it 'performs reasonably' do
+      start = Time.now
+      250.times do
+        lookup = ['ZM', 'ZMB', 'Zambia', 'US', 'USA', 'United States'].sample
+        case lookup.length
+        when 2 then ISO3166::Country.find_all_by(:alpha2, lookup)
+        when 3 then ISO3166::Country.find_all_by(:alpha3, lookup)
+        else ISO3166::Country.find_all_by(:name, lookup)
+        end
+      end
+      expect(Time.now - start).to be < 1
+    end
   end
 
   describe 'hash finder methods' do
@@ -798,6 +858,23 @@ describe ISO3166::Country do
     end
   end
 
+  describe 'in_esm?' do
+    let(:netherlands) { ISO3166::Country.search('NL') }
+    let(:switzerland) { ISO3166::Country.search('CH') }
+
+    it 'should return false for countries without esm_member or eea_member flag' do
+      expect(country.in_esm?).to be_falsey
+    end
+
+    it 'should return true for countries with eea_member flag set to true' do
+      expect(netherlands.in_esm?).to be_truthy
+    end
+
+    it 'should return true for countries with esm_member flag set to true' do
+      expect(switzerland.in_esm?).to be_truthy
+    end
+  end
+
   describe 'gec' do
     it 'should return the country\'s GEC code' do
       expect(ISO3166::Country.new('NA').gec).to eql 'WA'

data/spec/data_spec.rb

@@ -116,8 +116,8 @@ describe ISO3166::Data do
       expect(subject.name).to eq 'NEW Taiwan'
       expect(subject.translations).to eq('en' => 'NEW Taiwan',
                                          'de' => 'NEW Taiwan')
-      expect(subject.subdivisions).to eq(CHA: ISO3166::Subdivision.new({name: 'New Changhua'}),
-                                         CYI: ISO3166::Subdivision.new({name: 'New Municipality'}))
+      expect(subject.subdivisions).to eq('CHA' => ISO3166::Subdivision.new({name: 'New Changhua'}),
+                                         'CYI' => ISO3166::Subdivision.new({name: 'New Municipality'}))
     end
   end
 
@@ -143,8 +143,8 @@ describe ISO3166::Data do
       data = ISO3166::Data.new('LOL').call
       expect(data['name']).to eq 'Happy Country'
       expect(subject.name).to eq 'Happy Country'
-      expect(subject.subdivisions).to eq(LOL1: ISO3166::Subdivision.new({name: 'Happy sub1'}),
-                                         LOL2: ISO3166::Subdivision.new({name: 'Happy sub2'}))
+      expect(subject.subdivisions).to eq('LOL1' => ISO3166::Subdivision.new({name: 'Happy sub1'}),
+                                         'LOL2' => ISO3166::Subdivision.new({name: 'Happy sub2'}))
     end
 
     it 'detect a stale cache' do

data/spec/subdivision_spec.rb

@@ -3,6 +3,10 @@
 require 'spec_helper'
 
 describe ISO3166::Subdivision do
+  before do
+    ISO3166::Data.reset
+  end
+
   let(:countries) { ISO3166::Country.all }
   let(:available_types) { [Hash, NilClass] }
 
@@ -15,4 +19,14 @@ describe ISO3166::Subdivision do
       end
     end
   end
+
+  describe 'state codes' do
+    it 'should all be strings' do
+      countries.each do |country|
+        expect(country.subdivisions.keys).to all(be_a(String)), \
+          "Expected #{country.alpha2.inspect} to have string subdivision" \
+          "codes but had #{country.subdivisions.keys.inspect}"
+      end
+    end
+  end
 end

data/spec/thread_safety_spec.rb

@@ -0,0 +1,46 @@
+describe 'Accessing ISO3166::Country instances data in multiple threads' do
+  before do
+    if Thread.respond_to?(:report_on_exception)
+      @report_on_exception_value = Thread.report_on_exception
+      Thread.report_on_exception = false
+    end
+
+    ISO3166::Data.reset
+  end
+
+  def create_countries_threaded
+    nthreads = 100
+    threads = []
+
+    alpha2_codes = ['us', 'es', 'nl', 'ca', 'de', 'fr', 'mx', 'ru', 'ch', 'jp']
+
+    nthreads.times do
+      threads << Thread.new do
+        alpha2_codes.each do |a2|
+          country = ISO3166::Country[a2]
+          # This will fail if data['translations'] has been
+          # left nil due to a race condition
+          country.translation
+        end
+      end
+    end
+    threads.map(&:join)
+  end
+
+  it "doesn't raise any exceptions when using a mutex" do
+    expect { create_countries_threaded }.to_not raise_error
+  end
+
+  it "raises NoMethodError when not using a mutex" do
+    allow(ISO3166::Data).to receive(:use_mutex?).and_return(false)
+
+    expect { create_countries_threaded }.to raise_error(NoMethodError)
+  end
+
+  after do
+    if Thread.respond_to?(:report_on_exception)
+      Thread.report_on_exception = @report_on_exception_value
+    end
+  end
+end
+