twitter_cldr 6.6.0 → 6.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7840c80b8cc6bcf117de49198d83cdb18a938c57339f0facbef2bc67cef6a7c8
-  data.tar.gz: 8c92bead9ddba30899f7478dbcb94bed2d43cecb67fa69a5e565daff7930a02f
+  metadata.gz: eb3f9e357d9106f3b87744f5a8203eda30efd0dac40a24dacfbfaca6122431bc
+  data.tar.gz: a39e1373d65e2665124caedba55b039d62214950ced3db2e79636e38aef9e9e6
 SHA512:
-  metadata.gz: 144647b4f36e6ec2df0394978a0f975bcbfcc62f0799cf1080e9c058fcc6daea90d9ad5f2270c080f1422200abd34874ac44e92935783c16a4cfea22a6e3bfdd
-  data.tar.gz: 60a03fcc538428dbc767351fb0a2370e3bd85da5e3b239f3529e074730b9ae0c7f918f849753fbad5a9fce902982fe52aff0d1943919389b674f29bf09950ace
+  metadata.gz: 0e810877761c6898a5e5efb187c2f16a91536bf5e2bf784d802eb99ef9fe482140c79490f20e2c3748622147f340be3905cdc2cebc1c046e7eff9c5929130c8d
+  data.tar.gz: 2e9d84af9731ba6ada0cd0fc63224a54fbdb4f7de6b3de1d0f4748d20f71c6f17b65b490b5d32b56e269273c5c05dc94fa704da8fe89af077a62e50aad562023

data/Gemfile

@@ -4,7 +4,7 @@ gemspec
 
 group :development, :test do
   gem 'rake'
-  gem 'pry-nav'
+  gem 'pry-byebug' unless RUBY_PLATFORM == 'java'
   gem 'ruby-prof' unless RUBY_PLATFORM == 'java'
   gem 'regexp_parser', '~> 0.5'
   gem 'benchmark-ips'
@@ -13,7 +13,7 @@ group :development, :test do
   # gemspec allows any version, but most people are probably using 1.x, so
   # let's test and develop against that
   gem 'tzinfo', '< 2'
-  gem 'tzinfo-data', '= 1.2020.4'  # keep in sync with ICU
+  gem 'tzinfo-data', '= 1.2021.1'  # try to keep in sync with ICU
 end
 
 group :development do

data/README.md

@@ -1,4 +1,4 @@
-## twitter-cldr-rb [![Build Status](https://secure.travis-ci.org/twitter/twitter-cldr-rb.png?branch=master)](http://travis-ci.org/twitter/twitter-cldr-rb) [![Code Climate](https://codeclimate.com/github/twitter/twitter-cldr-rb.png)](https://codeclimate.com/github/twitter/twitter-cldr-rb) [![Coverage Status](https://coveralls.io/repos/twitter/twitter-cldr-rb/badge.png?branch=master)](https://coveralls.io/r/twitter/twitter-cldr-rb?branch=master)
+## twitter-cldr-rb ![Unit Tests](https://github.com/camertron/rux/actions/workflows/unit_tests.yml/badge.svg?branch=master) [![Code Climate](https://codeclimate.com/github/twitter/twitter-cldr-rb.png)](https://codeclimate.com/github/twitter/twitter-cldr-rb) [![Coverage Status](https://coveralls.io/repos/twitter/twitter-cldr-rb/badge.png?branch=master)](https://coveralls.io/r/twitter/twitter-cldr-rb?branch=master)
 
 TwitterCldr uses Unicode's Common Locale Data Repository (CLDR) to format certain types of text into their
 localized equivalents.  Currently supported types of text include dates, times, currencies, decimals, percentages, and symbols.
@@ -558,20 +558,20 @@ The CLDR contains postal code validation regexes for a number of countries.
 
 ```ruby
 # United States
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.valid?("94103")     # true
 postal_code.valid?("9410")      # false
 
 # England (Great Britain)
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:gb) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:gb)
 postal_code.valid?("BS98 1TL")  # true
 
 # Sweden
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:se) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:se)
 postal_code.valid?("280 12")    # true
 
 # Canada
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:ca) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:ca)
 postal_code.valid?("V3H 1Z7")   # true
 ```
 
@@ -579,7 +579,7 @@ Match all valid postal codes in a string with the `#find_all` method:
 
 ```ruby
 # United States
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.find_all("12345 23456")    # ["12345", "23456"]
 ```
 
@@ -592,7 +592,7 @@ TwitterCldr::Shared::PostalCodes.territories  # [:ac, :ad, :af, :ai, :al, ... ]
 Just want the regex?  No problem:
 
 ```ruby
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.regexp  # /(\d{5})(?:[ \-](\d{4}))?/
 ```
 
@@ -1014,17 +1014,17 @@ The Psych gem that is the default YAML engine in Ruby 1.9 doesn't handle Unicode
 You can make use of TwitterCLDR's YAML dumper by calling `localize` and then `to_yaml` on an `Array`, `Hash`, or `String`:
 
 ```ruby
-{ :hello => "world" }.localize.to_yaml 
-["hello", "world"].localize.to_yaml 
-"hello, world".localize.to_yaml 
+{ :hello => "world" }.localize.to_yaml
+["hello", "world"].localize.to_yaml
+"hello, world".localize.to_yaml
 ```
 
 Behind the scenes, these convenience methods are using the `TwitterCldr::Shared::YAML` class.  You can do the same thing if you're feeling adventurous:
 
 ```ruby
-TwitterCldr::Shared::YAML.dump({ :hello => "world" }) 
-TwitterCldr::Shared::YAML.dump(["hello", "world"]) 
-TwitterCldr::Shared::YAML.dump("hello, world") 
+TwitterCldr::Shared::YAML.dump({ :hello => "world" })
+TwitterCldr::Shared::YAML.dump(["hello", "world"])
+TwitterCldr::Shared::YAML.dump("hello, world")
 ```
 
 ## Adding New Locales

data/Rakefile

@@ -13,7 +13,7 @@ require 'rubygems/package_task'
 
 require './lib/twitter_cldr'
 
-require 'pry-nav'
+require 'pry-byebug' unless RUBY_PLATFORM == 'java'
 
 Bundler::GemHelper.install_tasks
 
@@ -89,6 +89,11 @@ namespace :update do
     TwitterCldr::Resources::NumberFormatsImporter.new.import
   end
 
+  desc 'Import list formats'
+  task :list_formats do
+    TwitterCldr::Resources::ListFormatsImporter.new.import
+  end
+
   desc 'Import currency symbols'
   task :currency_symbols do
     TwitterCldr::Resources::CurrencySymbolsImporter.new.import
@@ -136,6 +141,16 @@ namespace :update do
     TwitterCldr::Resources::UnicodePropertyAliasesImporter.new.import
   end
 
+  desc 'Import unit data'
+  task :units do
+    TwitterCldr::Resources::UnitsImporter.new.import
+  end
+
+  desc 'Import calendar data'
+  task :calendars do
+    TwitterCldr::Resources::CalendarsImporter.new.import
+  end
+
   desc 'Generate the casefolder class. Depends on unicode data'
   task :casefolder do
     TwitterCldr::Resources::CasefolderClassGenerator.new.import

data/lib/twitter_cldr/formatters/formatter.rb

@@ -26,10 +26,6 @@ module TwitterCldr
         token.value.gsub(/'([^']+)'/, '\1') # remove single-quote escaping for "real" characters
       end
 
-      def format_composite(token, index, obj, options)
-        eval(format(token.tokens, obj)).to_s
-      end
-
     end
   end
 end

data/lib/twitter_cldr/formatters/list_formatter.rb

@@ -62,7 +62,13 @@ module TwitterCldr
       end
 
       def resource_for_type(type)
-        resource[type]
+        loop do
+          if resource[type].is_a?(Symbol)
+            type = resource[:type].to_s.split('.').last
+          else
+            break resource[type]
+          end
+        end
       end
 
       def resource

data/lib/twitter_cldr/resources/calendars_importer.rb

@@ -0,0 +1,220 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'fileutils'
+require 'parallel'
+require 'etc'
+require 'set'
+
+module TwitterCldr
+  module Resources
+
+    class CalendarsImporter < Importer
+
+      requirement :cldr, Versions.cldr_version
+      output_path 'locales'
+      locales TwitterCldr.supported_locales
+      ruby_engine :mri
+
+      private
+
+      def execute
+        locales = Set.new
+
+        finish = -> (locale, *) do
+          locales.add(locale)
+          STDOUT.write "\rImported #{locale}, #{locales.size} of #{params[:locales].size} total"
+        end
+
+        Parallel.each(params[:locales], in_processes: Etc.nprocessors, finish: finish) do |locale|
+          import_locale(locale)
+          locales << locale
+        end
+      end
+
+      def import_locale(locale)
+        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+          GregorianCalendar.new(ancestor_locale, requirements[:cldr]).to_h
+        end
+
+        output_file = File.join(output_path, locale.to_s, 'calendars.yml')
+
+        File.open(output_file, 'w:utf-8') do |output|
+          output.write(
+            TwitterCldr::Utils::YAML.dump(
+              TwitterCldr::Utils.deep_symbolize_keys(locale => data),
+              use_natural_symbols: true
+            )
+          )
+        end
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+
+
+    class GregorianCalendar
+      attr_reader :locale, :cldr_req
+
+      def initialize(locale, cldr_req)
+        @locale = locale
+        @cldr_req = cldr_req
+      end
+
+      def to_h
+        {
+          calendars: {
+            gregorian: {
+              months:   contexts('month'),
+              days:     contexts('day'),
+              eras:     eras,
+              quarters: contexts('quarter'),
+              periods:  contexts('dayPeriod', group: "alt"),
+              fields:   fields,
+              formats: {
+                date:     formats('date'),
+                time:     formats('time'),
+                datetime: formats('dateTime')
+              },
+              additional_formats: additional_formats
+            }
+          }
+        }
+      end
+
+      private
+
+      def calendar
+        @calendar ||= doc.xpath('//ldml/dates/calendars/calendar[@type="gregorian"]').first
+      end
+
+      def contexts(kind, options = {})
+        return {} unless calendar
+
+        calendar.xpath("#{kind}s/#{kind}Context").each_with_object({}) do |node, result|
+          context = node.attribute('type').value.to_sym
+          result[context] = widths(node, kind, context, options)
+        end
+      end
+
+      def widths(node, kind, context, options = {})
+        node.xpath("#{kind}Width").each_with_object({}) do |node, result|
+          width = node.attribute('type').value.to_sym
+          result[width] = elements(node, kind, context, width, options)
+        end
+      end
+
+      def elements(node, kind, context, width, options = {})
+        aliased = node.xpath('alias').first
+
+        if aliased
+          alias_path = "#{node.path}/#{aliased.attribute('path').value}"
+          elements(doc.xpath(alias_path).first, kind, context, width, options)
+        else
+          node.xpath(kind).each_with_object({}) do |node, result|
+            key = node.attribute('type').value
+            key = key =~ /^\d*$/ ? key.to_i : key.to_sym
+
+            if options[:group] && found_group = node.attribute(options[:group])
+              result[found_group.value] ||= {}
+              result[found_group.value][key] = node.content
+            else
+              result[key] = node.content
+            end
+          end
+        end
+      end
+
+      def periods
+        am = calendar.xpath("am").first
+        pm = calendar.xpath("pm").first
+
+        {}.tap do |result|
+          result[:am] = am.content if am
+          result[:pm] = pm.content if pm
+        end
+      end
+
+      def eras
+        return {} unless calendar
+
+        base_path = "#{calendar.path}/eras"
+        keys = doc.xpath("#{base_path}/*").map { |node| node.name }
+
+        keys.each_with_object({}) do |name, result|
+          path = "#{base_path}/#{name}/*"
+          key  = name.gsub('era', '').gsub(/s$/, '').downcase.to_sym
+          result[key] = doc.xpath(path).each_with_object({}) do |node, ret|
+            type = node.attribute('type').value.to_i rescue 0
+            ret[type] = node.content
+            ret
+          end
+        end
+      end
+
+      def formats(type)
+        return {} unless calendar
+
+        formats = calendar.xpath("#{type}Formats/#{type}FormatLength").each_with_object({}) do |node, result|
+          key = node.attribute('type').value.to_sym rescue :format
+          result[key] = pattern(node, type)
+        end
+        if default = default_format(type)
+          formats = default.merge(formats)
+        end
+        formats
+      end
+
+      def additional_formats
+        return {} unless calendar
+
+        calendar.xpath("dateTimeFormats/availableFormats/dateFormatItem").each_with_object({}) do |node, result|
+          key = node.attribute('id').value
+          result[key] = node.content
+        end
+      end
+
+      def default_format(type)
+        if node = calendar.xpath("#{type}Formats/default").first
+          key = node.attribute('choice').value.to_sym
+          { :default => :"calendars.gregorian.formats.#{type.downcase}.#{key}" }
+        end
+      end
+
+      def pattern(node, type)
+        node.xpath("#{type}Format/pattern").each_with_object({}) do |node, result|
+          pattern = node.content
+          pattern = pattern.gsub('{0}', '{{time}}').gsub('{1}', '{{date}}') if type == 'dateTime'
+          result[:pattern] = pattern
+        end
+      end
+
+      def fields
+        doc.xpath("//ldml/dates/fields/field").each_with_object({}) do |node, result|
+          key  = node.attribute('type').value.to_sym
+          name = node.xpath('displayName').first
+          result[key] = name.content if name
+        end
+      end
+
+      def doc
+        @doc ||= begin
+          locale_fs = locale.to_s.gsub('-', '_')
+          Nokogiri.XML(File.read(File.join(cldr_main_path, "#{locale_fs}.xml")))
+        end
+      end
+
+      def cldr_main_path
+        @cldr_main_path ||= File.join(cldr_req.common_path, 'main')
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/language_codes_importer.rb

@@ -45,7 +45,7 @@ module TwitterCldr
           source_path = source_path_for(file)
 
           unless File.exist?(source_path)
-            open(source_path, 'wb') { |file| file << open(url).read }
+            open(source_path, 'wb') { |file| file << URI.open(url).read }
           end
         end
       end

data/lib/twitter_cldr/resources/list_formats_importer.rb

@@ -0,0 +1,140 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'parallel'
+require 'etc'
+require 'set'
+
+module TwitterCldr
+  module Resources
+
+    class ListFormatsImporter < Importer
+
+      requirement :cldr, Versions.cldr_version
+      output_path 'locales'
+      locales TwitterCldr.supported_locales
+      ruby_engine :mri
+
+      private
+
+      def execute
+        locales = Set.new
+
+        finish = -> (locale, *) do
+          locales.add(locale)
+          STDOUT.write "\rImported #{locale}, #{locales.size} of #{params[:locales].size} total"
+        end
+
+        Parallel.each(params[:locales], in_processes: Etc.nprocessors, finish: finish) do |locale|
+          import_locale(locale)
+          locales << locale
+        end
+
+        puts
+      end
+
+      def import_locale(locale)
+        # The merging that happens here works at the listPatternPart level of granularity.
+        # In other words, a missing part will be filled in by any part with the same key
+        # in the locale's ancestor chain. The raw CLDR data contains the inheritance marker
+        # (i.e. "↑↑↑") for listPatterns that are missing parts, but the expanded data we
+        # get in the downloadable CLDR zip file doesn't include them or the inherited data,
+        # making it impossible for TwitterCLDR to know how it should handle missing keys.
+        # I believe whatever massage tool the CLDR maintainers use to generate the final
+        # data set doesn't take aliases into account, which explains the holes in the data.
+        # By allowing individual listPatternParts to be populated by data from ancestor
+        # locales, we fill in any missing parts at the minor risk of being slightly wrong
+        # when formatting lists. In my opinion, it's far better to produce a slightly wrong
+        # string than to error or produce an entirely empty string.
+        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+          ListFormats.new(ancestor_locale, requirements[:cldr]).to_h
+        end
+
+        output_file = File.join(output_path, locale.to_s, 'lists.yml')
+
+        File.open(output_file, 'w:utf-8') do |output|
+          output.write(
+            TwitterCldr::Utils::YAML.dump(
+              TwitterCldr::Utils.deep_symbolize_keys(locale => data),
+              use_natural_symbols: true
+            )
+          )
+        end
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+
+
+    class ListFormats
+
+      attr_reader :locale, :cldr_req
+
+      def initialize(locale, cldr_req)
+        @locale = locale
+        @cldr_req = cldr_req
+      end
+
+      def to_h
+        { lists: lists }
+      end
+
+      def lists
+        doc.xpath('//ldml/listPatterns/listPattern').each_with_object({}) do |pattern_node, pattern_result|
+          pattern_type = if attribute = pattern_node.attribute('type')
+            attribute.value.to_sym
+          else
+            :default
+          end
+
+          pattern_node = pattern_for(pattern_type)
+
+          pattern_result[pattern_type] = pattern_node.xpath('listPatternPart').each_with_object({}) do |type_node, type_result|
+            type_result[type_node.attribute('type').value.to_sym] = type_node.content
+          end
+        end
+      end
+
+      def pattern_for(type)
+        xpath = xpath_for(type)
+        pattern_node = doc.xpath(xpath)[0]
+        alias_node = pattern_node.xpath('alias')[0]
+
+        if alias_node
+          alias_type = alias_node.attribute('path').value[/@type='([\w-]+)'/, 1] || :default
+          # follow aliases so we can fully expand them
+          pattern_node = pattern_for(alias_type)
+        end
+
+        pattern_node
+      end
+
+      def xpath_for(type)
+        if type == :default
+          '//ldml/listPatterns/listPattern[not(@type)]'
+        else
+          "//ldml/listPatterns/listPattern[@type='#{type}']"
+        end
+      end
+
+      def doc
+        @doc ||= begin
+          locale_fs = locale.to_s.gsub('-', '_')
+          Nokogiri.XML(File.read(File.join(cldr_main_path, "#{locale_fs}.xml")))
+        end
+      end
+
+      def cldr_main_path
+        @cldr_main_path ||= File.join(cldr_req.common_path, 'main')
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -22,13 +22,10 @@ module TwitterCldr
 
       LOCALE_COMPONENTS = %w[
         layout
-        calendars
         languages
         currencies
         plural_rules
-        lists
         rbnf
-        units
         fields
       ]
 
@@ -73,7 +70,6 @@ module TwitterCldr
           }
 
           Cldr::Export.export(export_args) do |component, locale, path|
-            add_buddhist_calendar(component, locale, path)
             process_plurals(component, locale, path)
             deep_symbolize(path)
           end
@@ -141,30 +137,6 @@ module TwitterCldr
           output.write(YAML.dump(locale => plurals))
         end
       end
-
-      # TODO: export buddhist calendar from CLDR data instead of using BUDDHIST_CALENDAR constant.
-      #
-      def add_buddhist_calendar(component, locale, path)
-        return unless component == 'Calendars' && locale == :th
-
-        data = YAML.load(File.read(path))
-        data['th']['calendars']['buddhist'] = BUDDHIST_CALENDAR
-
-        File.open(path, 'w:utf-8') { |output| output.write(YAML.dump(data))}
-      end
-
-      BUDDHIST_CALENDAR = {
-        'formats' => {
-          'date' => {
-            'default' => :'calendars.buddhist.formats.date.medium',
-            'full'    => { 'pattern' => 'EEEEที่ d MMMM G y' },
-            'long'    => { 'pattern' => 'd MMMM พ.ศ. #{y + 543}' },
-            'medium'  => { 'pattern' => 'd MMM #{y + 543}' },
-            'short'   => { 'pattern' => 'd/M/#{y + 543}' }
-          }
-        }
-      }
-
     end
 
   end

data/lib/twitter_cldr/resources/requirements/cldr_requirement.rb

@@ -4,8 +4,9 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'i18n'
+require 'net/ftp'
 require 'nokogiri'
-require 'open-uri'
+require 'tempfile'
 require 'uri'
 require 'zip'
 
@@ -14,7 +15,7 @@ module TwitterCldr
     module Requirements
 
       class CldrRequirement
-        CLDR_URL = "https://unicode.org/Public/cldr/%{version}/core.zip"
+        CLDR_URL = "ftp://unicode.org/Public/cldr/%{version}/core.zip".freeze
 
         attr_reader :version
 
@@ -111,15 +112,24 @@ module TwitterCldr
         end
 
         def download
-          URI.parse(cldr_url).open do |tempfile|
-            FileUtils.mkdir_p(source_path)
-            Zip.on_exists_proc = true
-
-            Zip::File.open(tempfile.path) do |file|
-              file.each do |entry|
-                path = File.join(source_path, entry.name)
-                FileUtils.mkdir_p(File.dirname(path))
-                file.extract(entry, path)
+          FileUtils.mkdir_p(source_path)
+
+          uri = URI(cldr_url)
+          ext_name = File.extname(cldr_url)
+          file_name = File.basename(cldr_url).chomp(ext_name)
+
+          Net::FTP.open(uri.host) do |ftp|
+            ftp.login
+
+            Tempfile.open([file_name, ext_name]) do |tmp|
+              ftp.getbinaryfile(uri.path, tmp.path)
+
+              Zip::File.open(tmp.path) do |file|
+                file.each do |entry|
+                  path = File.join(source_path, entry.name)
+                  FileUtils.mkdir_p(File.dirname(path))
+                  file.extract(entry, path)
+                end
               end
             end
           end

data/lib/twitter_cldr/resources/requirements/unicode_requirement.rb

@@ -3,7 +3,7 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'open-uri'
+require 'net/ftp'
 require 'fileutils'
 
 module TwitterCldr
@@ -45,8 +45,12 @@ module TwitterCldr
         def download(file)
           source_path = source_path_for(file)
           FileUtils.mkdir_p(File.dirname(source_path))
-          remote_url = File.join(url % { version: version }, file)
-          File.open(source_path, 'wb') { |file| file << URI.open(remote_url).read }
+          uri = URI(File.join(url % { version: version }, file))
+
+          Net::FTP.open(uri.host) do |ftp|
+            ftp.login
+            ftp.getbinaryfile(uri.path, source_path)
+          end
         end
       end