twitter_cldr 6.3.0 → 6.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e580aabc445c5e9873d2265fb966ea75f69112ddeb0a99e4cdea94593c3483c2
-  data.tar.gz: 07b10805c0e93f82f10a2413f9d982b1965e982dc72020877f59471586bcd24f
+  metadata.gz: 86de4e3cfb05e54b776d518c6a4c6decb9aaa36ec36ae2593e487cb12d7299ad
+  data.tar.gz: 8abd9399b1d44d30cd2cf818ef2c881b8dace4cf042b33e8a6086b02b5a0b561
 SHA512:
-  metadata.gz: 55e956abae69d8966471d7ac078d79c1cd638b12169afb5640e87989edac94b1b13a9afc6b295c2097b2ff285ff4ebbaa286c9549ca1379d46cb09527afa4564
-  data.tar.gz: 6781daa22d67537cd64dbf6d33fcc47e547fe8bf4f6ad883aa94354f567ba88a538cdb821d97a0918be329aae416ab94e303f5e03f04866f2162b31c67c3c5b0
+  metadata.gz: 57f9bb2095d152659a1a34f9a1c565496ddc6ac00cf5148f322709b5236d2c201502d7b9988e8ef7373ecff507b8fac944423d2c9a18dc8e65613fee9faf7440
+  data.tar.gz: 646cf95c956f2fa9655d4b96be566f4b59d18e5665ffe4bf04af3470aa0a75a9dbe1e163c74ed448eebed2b37ea98cd7e6c1b431e0b12c602a12a6cc47c438e4

data/Gemfile

@@ -4,7 +4,7 @@ gemspec
 
 group :development, :test do
   gem 'rake'
-  gem 'pry-nav'
+  gem 'pry-byebug'
   gem 'ruby-prof' unless RUBY_PLATFORM == 'java'
   gem 'regexp_parser', '~> 0.5'
   gem 'benchmark-ips'
@@ -13,7 +13,7 @@ group :development, :test do
   # gemspec allows any version, but most people are probably using 1.x, so
   # let's test and develop against that
   gem 'tzinfo', '< 2'
-  gem 'tzinfo-data', '= 1.2019.3'  # keep in sync with ICU
+  gem 'tzinfo-data', '= 1.2020.4'  # keep in sync with ICU
 end
 
 group :development do
@@ -22,7 +22,7 @@ group :development do
 
   gem 'ruby-cldr', github: 'camertron/ruby-cldr', branch: 'mapzones' # 'svenfuchs/ruby-cldr'
   gem 'i18n'
-  gem 'cldr-plurals', '~> 1.0'
+  gem 'cldr-plurals', '~> 1.1'
 end
 
 group :test do

data/README.md

@@ -1,4 +1,4 @@
-## twitter-cldr-rb [![Build Status](https://secure.travis-ci.org/twitter/twitter-cldr-rb.png?branch=master)](http://travis-ci.org/twitter/twitter-cldr-rb) [![Code Climate](https://codeclimate.com/github/twitter/twitter-cldr-rb.png)](https://codeclimate.com/github/twitter/twitter-cldr-rb) [![Coverage Status](https://coveralls.io/repos/twitter/twitter-cldr-rb/badge.png?branch=master)](https://coveralls.io/r/twitter/twitter-cldr-rb?branch=master)
+## twitter-cldr-rb ![Unit Tests](https://github.com/camertron/rux/actions/workflows/unit_tests.yml/badge.svg?branch=master) [![Code Climate](https://codeclimate.com/github/twitter/twitter-cldr-rb.png)](https://codeclimate.com/github/twitter/twitter-cldr-rb) [![Coverage Status](https://coveralls.io/repos/twitter/twitter-cldr-rb/badge.png?branch=master)](https://coveralls.io/r/twitter/twitter-cldr-rb?branch=master)
 
 TwitterCldr uses Unicode's Common Locale Data Repository (CLDR) to format certain types of text into their
 localized equivalents.  Currently supported types of text include dates, times, currencies, decimals, percentages, and symbols.
@@ -558,20 +558,20 @@ The CLDR contains postal code validation regexes for a number of countries.
 
 ```ruby
 # United States
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.valid?("94103")     # true
 postal_code.valid?("9410")      # false
 
 # England (Great Britain)
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:gb) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:gb)
 postal_code.valid?("BS98 1TL")  # true
 
 # Sweden
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:se) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:se)
 postal_code.valid?("280 12")    # true
 
 # Canada
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:ca) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:ca)
 postal_code.valid?("V3H 1Z7")   # true
 ```
 
@@ -579,7 +579,7 @@ Match all valid postal codes in a string with the `#find_all` method:
 
 ```ruby
 # United States
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.find_all("12345 23456")    # ["12345", "23456"]
 ```
 
@@ -592,7 +592,7 @@ TwitterCldr::Shared::PostalCodes.territories  # [:ac, :ad, :af, :ai, :al, ... ]
 Just want the regex?  No problem:
 
 ```ruby
-postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us)
 postal_code.regexp  # /(\d{5})(?:[ \-](\d{4}))?/
 ```
 
@@ -1014,17 +1014,17 @@ The Psych gem that is the default YAML engine in Ruby 1.9 doesn't handle Unicode
 You can make use of TwitterCLDR's YAML dumper by calling `localize` and then `to_yaml` on an `Array`, `Hash`, or `String`:
 
 ```ruby
-{ :hello => "world" }.localize.to_yaml 
-["hello", "world"].localize.to_yaml 
-"hello, world".localize.to_yaml 
+{ :hello => "world" }.localize.to_yaml
+["hello", "world"].localize.to_yaml
+"hello, world".localize.to_yaml
 ```
 
 Behind the scenes, these convenience methods are using the `TwitterCldr::Shared::YAML` class.  You can do the same thing if you're feeling adventurous:
 
 ```ruby
-TwitterCldr::Shared::YAML.dump({ :hello => "world" }) 
-TwitterCldr::Shared::YAML.dump(["hello", "world"]) 
-TwitterCldr::Shared::YAML.dump("hello, world") 
+TwitterCldr::Shared::YAML.dump({ :hello => "world" })
+TwitterCldr::Shared::YAML.dump(["hello", "world"])
+TwitterCldr::Shared::YAML.dump("hello, world")
 ```
 
 ## Adding New Locales

data/Rakefile

@@ -13,7 +13,7 @@ require 'rubygems/package_task'
 
 require './lib/twitter_cldr'
 
-require 'pry-nav'
+require 'pry-byebug'
 
 Bundler::GemHelper.install_tasks
 
@@ -89,6 +89,11 @@ namespace :update do
     TwitterCldr::Resources::NumberFormatsImporter.new.import
   end
 
+  desc 'Import list formats'
+  task :list_formats do
+    TwitterCldr::Resources::ListFormatsImporter.new.import
+  end
+
   desc 'Import currency symbols'
   task :currency_symbols do
     TwitterCldr::Resources::CurrencySymbolsImporter.new.import

data/lib/twitter_cldr/core_ext.rb

@@ -1,9 +1,10 @@
 # encoding: UTF-8
+require "bigdecimal"
 
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-[Integer, Float].each do |klass|
+[Integer, Float, BigDecimal].each do |klass|
   TwitterCldr::Localized::LocalizedNumber.localize(klass)
 end
 

data/lib/twitter_cldr/formatters/list_formatter.rb

@@ -62,7 +62,13 @@ module TwitterCldr
       end
 
       def resource_for_type(type)
-        resource[type]
+        loop do
+          if resource[type].is_a?(Symbol)
+            type = resource[:type].to_s.split('.').last
+          else
+            break resource[type]
+          end
+        end
       end
 
       def resource

data/lib/twitter_cldr/formatters/numbers/number_formatter.rb

@@ -69,24 +69,35 @@ module TwitterCldr
         precision = options[:precision] || precision_from(number)
         rounding = options[:rounding] || 0
 
-        number = "%.#{precision}f" % round_to(number, precision, rounding).abs
+        if number.is_a? BigDecimal
+          number = precision == 0 ?
+            round_to(number, precision, rounding).abs.fix.to_s("F") :
+            round_to(number, precision, rounding).abs.round(precision).to_s("F")
+        else
+          number = "%.#{precision}f" % round_to(number, precision, rounding).abs
+        end
         number.split(".")
       end
 
       def round_to(number, precision, rounding = 0)
         factor = 10 ** precision
-        result = (number * factor).round.to_f / factor
+        result = number.is_a?(BigDecimal) ?
+          ((number * factor).fix / factor) :
+          ((number * factor).round.to_f / factor)
 
         if rounding > 0
           rounding = rounding.to_f / factor
-          result = (result *  (1.0 / rounding)).round.to_f / (1.0 / rounding)
+          result = number.is_a?(BigDecimal) ?
+            ((result *  (1.0 / rounding)).fix / (1.0 / rounding)) :
+            ((result *  (1.0 / rounding)).round.to_f / (1.0 / rounding))
         end
 
         result
       end
 
       def precision_from(num)
-        parts = num.to_s.split(".")
+        return 0 if num.is_a?(BigDecimal) && num.fix == num
+        parts = (num.is_a?(BigDecimal) ? num.to_s("F") : num.to_s ).split(".")
         parts.size == 2 ? parts[1].size : 0
       end
 

data/lib/twitter_cldr/resources.rb

@@ -17,6 +17,7 @@ module TwitterCldr
     autoload :Importer,                       'twitter_cldr/resources/importer'
     autoload :ImportResolver,                 'twitter_cldr/resources/import_resolver'
     autoload :LanguageCodesImporter,          'twitter_cldr/resources/language_codes_importer'
+    autoload :ListFormatsImporter,            'twitter_cldr/resources/list_formats_importer'
     autoload :Loader,                         'twitter_cldr/resources/loader'
     autoload :LocalesResourcesImporter,       'twitter_cldr/resources/locales_resources_importer'
     autoload :NumberFormatsImporter,          'twitter_cldr/resources/number_formats_importer'
@@ -58,6 +59,7 @@ module TwitterCldr
           DayPeriodRulesImporter,
           HyphenationImporter,
           LanguageCodesImporter,
+          ListFormatsImporter,
           LocalesResourcesImporter,
           NumberFormatsImporter,
           ParentLocalesImporter,

data/lib/twitter_cldr/resources/list_formats_importer.rb

@@ -0,0 +1,140 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'parallel'
+require 'etc'
+require 'set'
+
+module TwitterCldr
+  module Resources
+
+    class ListFormatsImporter < Importer
+
+      requirement :cldr, Versions.cldr_version
+      output_path 'locales'
+      locales TwitterCldr.supported_locales
+      ruby_engine :mri
+
+      private
+
+      def execute
+        locales = Set.new
+
+        finish = -> (locale, *) do
+          locales.add(locale)
+          STDOUT.write "\rImported #{locale}, #{locales.size} of #{params[:locales].size} total"
+        end
+
+        Parallel.each(params[:locales], in_processes: Etc.nprocessors, finish: finish) do |locale|
+          import_locale(locale)
+          locales << locale
+        end
+
+        puts
+      end
+
+      def import_locale(locale)
+        # The merging that happens here works at the listPatternPart level of granularity.
+        # In other words, a missing part will be filled in by any part with the same key
+        # in the locale's ancestor chain. The raw CLDR data contains the inheritance marker
+        # (i.e. "↑↑↑") for listPatterns that are missing parts, but the expanded data we
+        # get in the downloadable CLDR zip file doesn't include them or the inherited data,
+        # making it impossible for TwitterCLDR to know how it should handle missing keys.
+        # I believe whatever massage tool the CLDR maintainers use to generate the final
+        # data set doesn't take aliases into account, which explains the holes in the data.
+        # By allowing individual listPatternParts to be populated by data from ancestor
+        # locales, we fill in any missing parts at the minor risk of being slightly wrong
+        # when formatting lists. In my opinion, it's far better to produce a slightly wrong
+        # string than to error or produce an entirely empty string.
+        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+          ListFormats.new(ancestor_locale, requirements[:cldr]).to_h
+        end
+
+        output_file = File.join(output_path, locale.to_s, 'lists.yml')
+
+        File.open(output_file, 'w:utf-8') do |output|
+          output.write(
+            TwitterCldr::Utils::YAML.dump(
+              TwitterCldr::Utils.deep_symbolize_keys(locale => data),
+              use_natural_symbols: true
+            )
+          )
+        end
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+
+
+    class ListFormats
+
+      attr_reader :locale, :cldr_req
+
+      def initialize(locale, cldr_req)
+        @locale = locale
+        @cldr_req = cldr_req
+      end
+
+      def to_h
+        { lists: lists }
+      end
+
+      def lists
+        doc.xpath('//ldml/listPatterns/listPattern').each_with_object({}) do |pattern_node, pattern_result|
+          pattern_type = if attribute = pattern_node.attribute('type')
+            attribute.value.to_sym
+          else
+            :default
+          end
+
+          pattern_node = pattern_for(pattern_type)
+
+          pattern_result[pattern_type] = pattern_node.xpath('listPatternPart').each_with_object({}) do |type_node, type_result|
+            type_result[type_node.attribute('type').value.to_sym] = type_node.content
+          end
+        end
+      end
+
+      def pattern_for(type)
+        xpath = xpath_for(type)
+        pattern_node = doc.xpath(xpath)[0]
+        alias_node = pattern_node.xpath('alias')[0]
+
+        if alias_node
+          alias_type = alias_node.attribute('path').value[/@type='([\w-]+)'/, 1] || :default
+          # follow aliases so we can fully expand them
+          pattern_node = pattern_for(alias_type)
+        end
+
+        pattern_node
+      end
+
+      def xpath_for(type)
+        if type == :default
+          '//ldml/listPatterns/listPattern[not(@type)]'
+        else
+          "//ldml/listPatterns/listPattern[@type='#{type}']"
+        end
+      end
+
+      def doc
+        @doc ||= begin
+          locale_fs = locale.to_s.gsub('-', '_')
+          Nokogiri.XML(File.read(File.join(cldr_main_path, "#{locale_fs}.xml")))
+        end
+      end
+
+      def cldr_main_path
+        @cldr_main_path ||= File.join(cldr_req.common_path, 'main')
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -26,7 +26,6 @@ module TwitterCldr
         languages
         currencies
         plural_rules
-        lists
         rbnf
         units
         fields

data/lib/twitter_cldr/resources/segment_rules_importer.rb

@@ -138,7 +138,7 @@ module TwitterCldr
 
             # this should be the last entry, but for some reason ICU returns
             # one more out-of-order range past the Unicode max
-            break if range.endCodePoint == 0x10FFFF
+            break if range.getEnd == 0x10FFFF
           end
         end
 
@@ -147,7 +147,7 @@ module TwitterCldr
       end
 
       def range_to_a(range)
-        [range.startCodePoint, range.endCodePoint, range.value]
+        [range.getStart, range.getEnd, range.getValue]
       end
 
       def rbbi_data_for(kind, locale)

data/lib/twitter_cldr/segmentation/state_machine.rb

@@ -13,8 +13,9 @@ module TwitterCldr
 
       START_STATE = 1
       STOP_STATE = 0
-      NEXT_STATES = 4
+      NEXT_STATES = 3
       ACCEPTING = 0
+      ACCEPTING_UNCONDITIONAL = 1
 
       class << self
         def instance(boundary_type, locale)
@@ -70,7 +71,7 @@ module TwitterCldr
       def handle_next(cursor)
         result = initial_position = cursor.position
         state = START_STATE
-        row = state * (metadata.category_count + 4)
+        row = state * (metadata.category_count + NEXT_STATES)
         category = 3
         mode = :run
 
@@ -86,20 +87,15 @@ module TwitterCldr
             category = 1
           elsif mode == :run
             category = category_table.get(cursor.codepoint)
-
-            if (category & 0x4000) != 0
-              category &= ~0x4000
-            end
-
             cursor.advance
           else
             mode = :run
           end
 
           state = ftable[row + NEXT_STATES + category]
-          row = state * (metadata.category_count + 4)
+          row = state * (metadata.category_count + NEXT_STATES)
 
-          if ftable[row + ACCEPTING] == -1
+          if ftable[row + ACCEPTING] == ACCEPTING_UNCONDITIONAL
             # match found
             result = cursor.position
           end

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = '6.3.0'
+  VERSION = '6.6.2'
 end

data/lib/twitter_cldr/versions.rb

@@ -5,8 +5,8 @@
 
 module TwitterCldr
   module Versions
-    CLDR_VERSION    = '37'
-    ICU_VERSION     = '67.1'
+    CLDR_VERSION    = '38.1'
+    ICU_VERSION     = '68.2'
     UNICODE_VERSION = '13.0.0'
 
     class << self