twitter_cldr 6.6.1 → 6.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46779462152c01a424c966b9a2d8a4d7b3d50c41164fd054edc493f96ddef7f5
-  data.tar.gz: 20689d92c66c2a42b1bb9545495b789460a4b892f8fd6b187f9967a459467a00
+  metadata.gz: c8aceda15c987295a62672bd4f7a78546b34eed2a83eb9178b58bd5a5efa49be
+  data.tar.gz: b179d63e855785bdd871b12dda0f7c0ecd67b91106510972d89f74ab93b8ec5d
 SHA512:
-  metadata.gz: dc9567be97d3604bfcef46520704ce3d05e2a5f2916e857791cc76de2f67a45a9aa014c6b63327a34caf89e4681696664741037ccf57b5988a30ff5891173a7f
-  data.tar.gz: 99a1b37964339e2d77c9fb295a7269025d6ef6ab81f9514190d55644d96262b848e21f48e84746c7ccb05f138bf0ef611e132de5ad896c67fdfe3091a1ece924
+  metadata.gz: 92f18ec12c13f8ed66b23a9453c9b932e502738266957b326a62493ef1062878ae29932d16f4963658f0b30591528de8415293f1e1c90e394a3e2b4fba2b1244
+  data.tar.gz: b34f99bbb5e5e37d3c95d63ce56d47c097d2aac14730ed56d72fc2f5e3c5a0db26b1ff0d20ea599a200d4e75bb3c38cd22ecf5321ece0a606967f0f49290738d

data/Gemfile

@@ -4,7 +4,7 @@ gemspec
 
 group :development, :test do
   gem 'rake'
-  gem 'pry-byebug'
+  gem 'pry-byebug' unless RUBY_PLATFORM == 'java'
   gem 'ruby-prof' unless RUBY_PLATFORM == 'java'
   gem 'regexp_parser', '~> 0.5'
   gem 'benchmark-ips'
@@ -13,7 +13,7 @@ group :development, :test do
   # gemspec allows any version, but most people are probably using 1.x, so
   # let's test and develop against that
   gem 'tzinfo', '< 2'
-  gem 'tzinfo-data', '= 1.2020.4'  # keep in sync with ICU
+  gem 'tzinfo-data', '= 1.2021.1'  # try to keep in sync with ICU
 end
 
 group :development do

data/Rakefile

@@ -13,7 +13,7 @@ require 'rubygems/package_task'
 
 require './lib/twitter_cldr'
 
-require 'pry-byebug'
+require 'pry-byebug' unless RUBY_PLATFORM == 'java'
 
 Bundler::GemHelper.install_tasks
 
@@ -141,6 +141,16 @@ namespace :update do
     TwitterCldr::Resources::UnicodePropertyAliasesImporter.new.import
   end
 
+  desc 'Import unit data'
+  task :units do
+    TwitterCldr::Resources::UnitsImporter.new.import
+  end
+
+  desc 'Import calendar data'
+  task :calendars do
+    TwitterCldr::Resources::CalendarsImporter.new.import
+  end
+
   desc 'Generate the casefolder class. Depends on unicode data'
   task :casefolder do
     TwitterCldr::Resources::CasefolderClassGenerator.new.import

data/lib/twitter_cldr/formatters/formatter.rb

@@ -26,10 +26,6 @@ module TwitterCldr
         token.value.gsub(/'([^']+)'/, '\1') # remove single-quote escaping for "real" characters
       end
 
-      def format_composite(token, index, obj, options)
-        eval(format(token.tokens, obj)).to_s
-      end
-
     end
   end
 end

data/lib/twitter_cldr/formatters/list_formatter.rb

@@ -62,7 +62,13 @@ module TwitterCldr
       end
 
       def resource_for_type(type)
-        resource[type]
+        loop do
+          if resource[type].is_a?(Symbol)
+            type = resource[:type].to_s.split('.').last
+          else
+            break resource[type]
+          end
+        end
       end
 
       def resource

data/lib/twitter_cldr/resources/calendars_importer.rb

@@ -0,0 +1,220 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'fileutils'
+require 'parallel'
+require 'etc'
+require 'set'
+
+module TwitterCldr
+  module Resources
+
+    class CalendarsImporter < Importer
+
+      requirement :cldr, Versions.cldr_version
+      output_path 'locales'
+      locales TwitterCldr.supported_locales
+      ruby_engine :mri
+
+      private
+
+      def execute
+        locales = Set.new
+
+        finish = -> (locale, *) do
+          locales.add(locale)
+          STDOUT.write "\rImported #{locale}, #{locales.size} of #{params[:locales].size} total"
+        end
+
+        Parallel.each(params[:locales], in_processes: Etc.nprocessors, finish: finish) do |locale|
+          import_locale(locale)
+          locales << locale
+        end
+      end
+
+      def import_locale(locale)
+        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+          GregorianCalendar.new(ancestor_locale, requirements[:cldr]).to_h
+        end
+
+        output_file = File.join(output_path, locale.to_s, 'calendars.yml')
+
+        File.open(output_file, 'w:utf-8') do |output|
+          output.write(
+            TwitterCldr::Utils::YAML.dump(
+              TwitterCldr::Utils.deep_symbolize_keys(locale => data),
+              use_natural_symbols: true
+            )
+          )
+        end
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+
+
+    class GregorianCalendar
+      attr_reader :locale, :cldr_req
+
+      def initialize(locale, cldr_req)
+        @locale = locale
+        @cldr_req = cldr_req
+      end
+
+      def to_h
+        {
+          calendars: {
+            gregorian: {
+              months:   contexts('month'),
+              days:     contexts('day'),
+              eras:     eras,
+              quarters: contexts('quarter'),
+              periods:  contexts('dayPeriod', group: "alt"),
+              fields:   fields,
+              formats: {
+                date:     formats('date'),
+                time:     formats('time'),
+                datetime: formats('dateTime')
+              },
+              additional_formats: additional_formats
+            }
+          }
+        }
+      end
+
+      private
+
+      def calendar
+        @calendar ||= doc.xpath('//ldml/dates/calendars/calendar[@type="gregorian"]').first
+      end
+
+      def contexts(kind, options = {})
+        return {} unless calendar
+
+        calendar.xpath("#{kind}s/#{kind}Context").each_with_object({}) do |node, result|
+          context = node.attribute('type').value.to_sym
+          result[context] = widths(node, kind, context, options)
+        end
+      end
+
+      def widths(node, kind, context, options = {})
+        node.xpath("#{kind}Width").each_with_object({}) do |node, result|
+          width = node.attribute('type').value.to_sym
+          result[width] = elements(node, kind, context, width, options)
+        end
+      end
+
+      def elements(node, kind, context, width, options = {})
+        aliased = node.xpath('alias').first
+
+        if aliased
+          alias_path = "#{node.path}/#{aliased.attribute('path').value}"
+          elements(doc.xpath(alias_path).first, kind, context, width, options)
+        else
+          node.xpath(kind).each_with_object({}) do |node, result|
+            key = node.attribute('type').value
+            key = key =~ /^\d*$/ ? key.to_i : key.to_sym
+
+            if options[:group] && found_group = node.attribute(options[:group])
+              result[found_group.value] ||= {}
+              result[found_group.value][key] = node.content
+            else
+              result[key] = node.content
+            end
+          end
+        end
+      end
+
+      def periods
+        am = calendar.xpath("am").first
+        pm = calendar.xpath("pm").first
+
+        {}.tap do |result|
+          result[:am] = am.content if am
+          result[:pm] = pm.content if pm
+        end
+      end
+
+      def eras
+        return {} unless calendar
+
+        base_path = "#{calendar.path}/eras"
+        keys = doc.xpath("#{base_path}/*").map { |node| node.name }
+
+        keys.each_with_object({}) do |name, result|
+          path = "#{base_path}/#{name}/*"
+          key  = name.gsub('era', '').gsub(/s$/, '').downcase.to_sym
+          result[key] = doc.xpath(path).each_with_object({}) do |node, ret|
+            type = node.attribute('type').value.to_i rescue 0
+            ret[type] = node.content
+            ret
+          end
+        end
+      end
+
+      def formats(type)
+        return {} unless calendar
+
+        formats = calendar.xpath("#{type}Formats/#{type}FormatLength").each_with_object({}) do |node, result|
+          key = node.attribute('type').value.to_sym rescue :format
+          result[key] = pattern(node, type)
+        end
+        if default = default_format(type)
+          formats = default.merge(formats)
+        end
+        formats
+      end
+
+      def additional_formats
+        return {} unless calendar
+
+        calendar.xpath("dateTimeFormats/availableFormats/dateFormatItem").each_with_object({}) do |node, result|
+          key = node.attribute('id').value
+          result[key] = node.content
+        end
+      end
+
+      def default_format(type)
+        if node = calendar.xpath("#{type}Formats/default").first
+          key = node.attribute('choice').value.to_sym
+          { :default => :"calendars.gregorian.formats.#{type.downcase}.#{key}" }
+        end
+      end
+
+      def pattern(node, type)
+        node.xpath("#{type}Format/pattern").each_with_object({}) do |node, result|
+          pattern = node.content
+          pattern = pattern.gsub('{0}', '{{time}}').gsub('{1}', '{{date}}') if type == 'dateTime'
+          result[:pattern] = pattern
+        end
+      end
+
+      def fields
+        doc.xpath("//ldml/dates/fields/field").each_with_object({}) do |node, result|
+          key  = node.attribute('type').value.to_sym
+          name = node.xpath('displayName').first
+          result[key] = name.content if name
+        end
+      end
+
+      def doc
+        @doc ||= begin
+          locale_fs = locale.to_s.gsub('-', '_')
+          Nokogiri.XML(File.read(File.join(cldr_main_path, "#{locale_fs}.xml")))
+        end
+      end
+
+      def cldr_main_path
+        @cldr_main_path ||= File.join(cldr_req.common_path, 'main')
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/language_codes_importer.rb

@@ -45,7 +45,7 @@ module TwitterCldr
           source_path = source_path_for(file)
 
           unless File.exist?(source_path)
-            open(source_path, 'wb') { |file| file << open(url).read }
+            open(source_path, 'wb') { |file| file << URI.open(url).read }
           end
         end
       end

data/lib/twitter_cldr/resources/list_formats_importer.rb

@@ -32,9 +32,23 @@ module TwitterCldr
           import_locale(locale)
           locales << locale
         end
+
+        puts
       end
 
       def import_locale(locale)
+        # The merging that happens here works at the listPatternPart level of granularity.
+        # In other words, a missing part will be filled in by any part with the same key
+        # in the locale's ancestor chain. The raw CLDR data contains the inheritance marker
+        # (i.e. "↑↑↑") for listPatterns that are missing parts, but the expanded data we
+        # get in the downloadable CLDR zip file doesn't include them or the inherited data,
+        # making it impossible for TwitterCLDR to know how it should handle missing keys.
+        # I believe whatever massage tool the CLDR maintainers use to generate the final
+        # data set doesn't take aliases into account, which explains the holes in the data.
+        # By allowing individual listPatternParts to be populated by data from ancestor
+        # locales, we fill in any missing parts at the minor risk of being slightly wrong
+        # when formatting lists. In my opinion, it's far better to produce a slightly wrong
+        # string than to error or produce an entirely empty string.
         data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
           ListFormats.new(ancestor_locale, requirements[:cldr]).to_h
         end
@@ -79,11 +93,7 @@ module TwitterCldr
             :default
           end
 
-          if aliased = pattern_node.xpath('alias').first
-            alias_type = aliased.attribute('path').value[/@type='([\w-]+)'/, 1]
-            pattern_result[pattern_type] = :"lists.#{alias_type || 'default'}"
-            next
-          end
+          pattern_node = pattern_for(pattern_type)
 
           pattern_result[pattern_type] = pattern_node.xpath('listPatternPart').each_with_object({}) do |type_node, type_result|
             type_result[type_node.attribute('type').value.to_sym] = type_node.content
@@ -91,6 +101,28 @@ module TwitterCldr
         end
       end
 
+      def pattern_for(type)
+        xpath = xpath_for(type)
+        pattern_node = doc.xpath(xpath)[0]
+        alias_node = pattern_node.xpath('alias')[0]
+
+        if alias_node
+          alias_type = alias_node.attribute('path').value[/@type='([\w-]+)'/, 1] || :default
+          # follow aliases so we can fully expand them
+          pattern_node = pattern_for(alias_type)
+        end
+
+        pattern_node
+      end
+
+      def xpath_for(type)
+        if type == :default
+          '//ldml/listPatterns/listPattern[not(@type)]'
+        else
+          "//ldml/listPatterns/listPattern[@type='#{type}']"
+        end
+      end
+
       def doc
         @doc ||= begin
           locale_fs = locale.to_s.gsub('-', '_')

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -22,12 +22,10 @@ module TwitterCldr
 
       LOCALE_COMPONENTS = %w[
         layout
-        calendars
         languages
         currencies
         plural_rules
         rbnf
-        units
         fields
       ]
 
@@ -72,7 +70,6 @@ module TwitterCldr
           }
 
           Cldr::Export.export(export_args) do |component, locale, path|
-            add_buddhist_calendar(component, locale, path)
             process_plurals(component, locale, path)
             deep_symbolize(path)
           end
@@ -140,30 +137,6 @@ module TwitterCldr
           output.write(YAML.dump(locale => plurals))
         end
       end
-
-      # TODO: export buddhist calendar from CLDR data instead of using BUDDHIST_CALENDAR constant.
-      #
-      def add_buddhist_calendar(component, locale, path)
-        return unless component == 'Calendars' && locale == :th
-
-        data = YAML.load(File.read(path))
-        data['th']['calendars']['buddhist'] = BUDDHIST_CALENDAR
-
-        File.open(path, 'w:utf-8') { |output| output.write(YAML.dump(data))}
-      end
-
-      BUDDHIST_CALENDAR = {
-        'formats' => {
-          'date' => {
-            'default' => :'calendars.buddhist.formats.date.medium',
-            'full'    => { 'pattern' => 'EEEEที่ d MMMM G y' },
-            'long'    => { 'pattern' => 'd MMMM พ.ศ. #{y + 543}' },
-            'medium'  => { 'pattern' => 'd MMM #{y + 543}' },
-            'short'   => { 'pattern' => 'd/M/#{y + 543}' }
-          }
-        }
-      }
-
     end
 
   end

data/lib/twitter_cldr/resources/requirements/cldr_requirement.rb

@@ -4,8 +4,9 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'i18n'
+require 'net/ftp'
 require 'nokogiri'
-require 'open-uri'
+require 'tempfile'
 require 'uri'
 require 'zip'
 
@@ -14,7 +15,7 @@ module TwitterCldr
     module Requirements
 
       class CldrRequirement
-        CLDR_URL = "https://unicode.org/Public/cldr/%{version}/core.zip"
+        CLDR_URL = "ftp://unicode.org/Public/cldr/%{version}/core.zip".freeze
 
         attr_reader :version
 
@@ -111,15 +112,24 @@ module TwitterCldr
         end
 
         def download
-          URI.parse(cldr_url).open do |tempfile|
-            FileUtils.mkdir_p(source_path)
-            Zip.on_exists_proc = true
-
-            Zip::File.open(tempfile.path) do |file|
-              file.each do |entry|
-                path = File.join(source_path, entry.name)
-                FileUtils.mkdir_p(File.dirname(path))
-                file.extract(entry, path)
+          FileUtils.mkdir_p(source_path)
+
+          uri = URI(cldr_url)
+          ext_name = File.extname(cldr_url)
+          file_name = File.basename(cldr_url).chomp(ext_name)
+
+          Net::FTP.open(uri.host) do |ftp|
+            ftp.login
+
+            Tempfile.open([file_name, ext_name]) do |tmp|
+              ftp.getbinaryfile(uri.path, tmp.path)
+
+              Zip::File.open(tmp.path) do |file|
+                file.each do |entry|
+                  path = File.join(source_path, entry.name)
+                  FileUtils.mkdir_p(File.dirname(path))
+                  file.extract(entry, path)
+                end
               end
             end
           end

data/lib/twitter_cldr/resources/requirements/unicode_requirement.rb

@@ -3,7 +3,7 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'open-uri'
+require 'net/ftp'
 require 'fileutils'
 
 module TwitterCldr
@@ -45,8 +45,12 @@ module TwitterCldr
         def download(file)
           source_path = source_path_for(file)
           FileUtils.mkdir_p(File.dirname(source_path))
-          remote_url = File.join(url % { version: version }, file)
-          File.open(source_path, 'wb') { |file| file << URI.open(remote_url).read }
+          uri = URI(File.join(url % { version: version }, file))
+
+          Net::FTP.open(uri.host) do |ftp|
+            ftp.login
+            ftp.getbinaryfile(uri.path, source_path)
+          end
         end
       end
 

data/lib/twitter_cldr/resources/segment_dictionaries_importer.rb

@@ -9,8 +9,7 @@ require 'open-uri'
 module TwitterCldr
   module Resources
     class SegmentDictionariesImporter < Importer
-
-      URL_TEMPLATE = 'https://raw.githubusercontent.com/unicode-org/icu/%{icu_version}/%{path}'
+      URL_TEMPLATE = 'https://raw.githubusercontent.com/unicode-org/icu/%{tag}/%{path}'
 
       DICTIONARY_FILES = [
         'icu4c/source/data/brkitr/dictionaries/burmesedict.txt',
@@ -57,11 +56,15 @@ module TwitterCldr
 
       def url_for(dictionary_file)
         URL_TEMPLATE % {
-          icu_version: "release-#{Versions.icu_version.gsub('.', '-')}",
+          tag: tag,
           path: dictionary_file
         }
       end
 
+      def tag
+        @tag ||= "release-#{Versions.icu_version.gsub('.', '-')}"
+      end
+
       def output_path_for(dictionary_file)
         file = File.basename(dictionary_file).chomp(File.extname(dictionary_file))
         File.join(output_path, "#{file}.dump")

data/lib/twitter_cldr/resources/timezone_tests_importer.rb

@@ -58,6 +58,14 @@ module TwitterCldr
       private
 
       def check_tzdata_versions
+        puts(
+          "Time zone data version check disabled. If you're upgrading to a new "\
+          "version of CLDR and ICU, please consider re-enabling this check in "\
+          "#{__FILE__} on line #{__LINE__ - 4}"
+        )
+
+        return
+
         resource_bundle = requirements[:icu].get_class('com.ibm.icu.util.UResourceBundle')
         icu_data = requirements[:icu].get_class('com.ibm.icu.impl.ICUData')
         zone_info_res_field = requirements[:icu]

data/lib/twitter_cldr/resources/units_importer.rb

@@ -0,0 +1,134 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'fileutils'
+require 'parallel'
+require 'etc'
+require 'set'
+
+module TwitterCldr
+  module Resources
+
+    class UnitsImporter < Importer
+
+      requirement :cldr, Versions.cldr_version
+      output_path 'locales'
+      locales TwitterCldr.supported_locales
+      ruby_engine :mri
+
+      private
+
+      def execute
+        locales = Set.new
+
+        finish = -> (locale, *) do
+          locales.add(locale)
+          STDOUT.write "\rImported #{locale}, #{locales.size} of #{params[:locales].size} total"
+        end
+
+        Parallel.each(params[:locales], in_processes: Etc.nprocessors, finish: finish) do |locale|
+          import_locale(locale)
+          locales << locale
+        end
+      end
+
+      def import_locale(locale)
+        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+          Units.new(ancestor_locale, requirements[:cldr]).to_h
+        end
+
+        output_file = File.join(output_path, locale.to_s, 'units.yml')
+
+        File.open(output_file, 'w:utf-8') do |output|
+          output.write(
+            TwitterCldr::Utils::YAML.dump(
+              TwitterCldr::Utils.deep_symbolize_keys(locale => data),
+              use_natural_symbols: true
+            )
+          )
+        end
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+
+
+    class Units
+      attr_reader :locale, :cldr_req
+
+      def initialize(locale, cldr_req)
+        @locale = locale
+        @cldr_req = cldr_req
+      end
+
+      def to_h
+        {
+          units: {
+            unitLength: unit_length,
+            durationUnit: duration_unit
+          }
+        }
+      end
+
+      private
+
+      def unit_length
+        doc.xpath('//ldml/units/unitLength').each_with_object({}) do |node, result|
+          type = node.attribute('type').value.to_sym
+          result[type] = units(node)
+        end
+      end
+
+      def resolve_unit_node(root, unit_node)
+        alias_node = unit_node.xpath('alias')[0]
+        return unit_node unless alias_node
+
+        # follow aliases so we can fully expand them
+        alias_type = alias_node.attribute('path').value[/@type='([\w-]+)'/, 1]
+        found_node = root.xpath("unit[@type='#{alias_type}']")
+        resolve_unit_node(root, found_node)
+      end
+
+      def units(node)
+        node.xpath('unit').each_with_object({}) do |unit_node, result|
+          unit_node = resolve_unit_node(node, unit_node)
+          type = unit_node.attribute('type').value.to_sym
+          found_unit = unit(unit_node)
+          result[type] = found_unit unless found_unit.empty?
+        end
+      end
+
+      def unit(node)
+        node.xpath('unitPattern').each_with_object({}) do |node, result|
+          count = node.attribute('count') ? node.attribute('count').value.to_sym : :one
+          result[count] = node.content
+        end
+      end
+
+      def duration_unit
+        doc.xpath('//ldml/units/durationUnit').each_with_object({}) do |node, result|
+          result[node.attribute('type').value.to_sym] = node.xpath('durationUnitPattern').first.content
+        end
+      end
+
+      def doc
+        @doc ||= begin
+          locale_fs = locale.to_s.gsub('-', '_')
+          Nokogiri.XML(File.read(File.join(cldr_main_path, "#{locale_fs}.xml")))
+        end
+      end
+
+      def cldr_main_path
+        @cldr_main_path ||= File.join(cldr_req.common_path, 'main')
+      end
+
+    end
+
+  end
+end