twitter_cldr 3.0.3 → 3.0.4

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 3724c4298d198dc34908dd336418c19edde6f09e
-  data.tar.gz: 72c50812ce644c70245b774155212becde60020e
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODMyYmYzMDM4ODg4NjY1OTIzOGMxNDY1MTRkNWI5YzU2Njg4ZDI1Yg==
+  data.tar.gz: !binary |-
+    N2YxZWZlMDNhZDRjODI3NGU1OGZlZWVhZjNhMDk0NzBhYjQ3ZGU5Mg==
 SHA512:
-  metadata.gz: 4c11d5870b7b355a40cff2348892eabac422f33a01e28ba5db006af38b59e8543fa97d42b9f516ad8e87f18358eb258338e7cd7aa27df9687f5dd583cfe28c9e
-  data.tar.gz: 3ef234d9ff5a630854d83010f63be2302d60ff130b75577ed6bd5a8cf4000bb44277dcd464ac2023960aa079ea81477695d0b9593f79355c5d59b76e77e9d599
+  metadata.gz: !binary |-
+    OGE2MTZmYTZhYTg2ZDBiZjA5MDk0YWQ1YTRlZjEzYjA5MzQ1ZDI2Y2IxYzg5
+    MjlhYjdhYmVkMzUyN2M4ZmU2NTljNzQ1YTUwZWIyNzFjYTYyODk1ZDdiZDA3
+    NzY4ZGJlNDIxOTdkYjhlMjRiMzNkYmQ5Njg0NDcxZjAzY2RiYjU=
+  data.tar.gz: !binary |-
+    ODM4YjY1MmI1MzQ2MWRmZTZkYjIyYTk1MTU0NmU4ODNhNDAyZTRjMzJiY2Yw
+    MDM3NGUwNmNiODY4NGIwNjA1YmRhOGQ4YzNiZjE2YzU5OGE0NGE2NDlmMGIy
+    NmI3NGZiMTM5ODk0YWI5YThhZDBiYWM3NzBmODhmYTBkNzk5MjE=

data/Gemfile

@@ -5,10 +5,8 @@ gemspec
 group :development, :test do
   gem 'rake'
 
-  platform :mri do
-    gem 'pry'
-    gem 'pry-nav'
-  end
+  gem 'pry'
+  gem 'pry-nav'
 
   platform :rbx do
     gem 'rubinius-debugger'
@@ -28,9 +26,7 @@ end
 group :development do
   gem 'nokogiri', "~> 1.5.9"
 
-  # https://github.com/svenfuchs/ruby-cldr/pull/18
-  # gem 'ruby-cldr', :github => 'svenfuchs/ruby-cldr'
-  gem 'ruby-cldr', :github => 'camertron/ruby-cldr', :branch => "segmentation"
+  gem 'ruby-cldr', :github => 'svenfuchs/ruby-cldr', :tag => 'v0.1.1'
 end
 
 group :test do

data/History.txt

@@ -1,3 +1,9 @@
+== 3.0.4
+
+* Fixed short numbers formatting for ja, ko, af, and a few other locales.
+* Added more locales: de-CH, en-AU, en-CA, en-GB, en-IE, en-SG, en-ZA,
+  es-CO, es-MX, es-US, fr-BE, fr-CA, fr-CH, it-CH.
+
 == 3.0.3
 
 * Rubinius support.

data/README.md

@@ -472,7 +472,7 @@ postal_code.regexp  # /\d{5}([ \-]\d{4})?/
 Get a sample of valid postal codes with the `#sample` method:
 
 ```ruby
-postal_code.sample(5)  # ["50043", "13363-8821", "70461", "34021-2280", "82049"]
+postal_code.sample(5)  # ["15175", "65142", "23110", "81366", "81295-6386"]
 ```
 
 ### Phone Codes

data/Rakefile

@@ -62,22 +62,30 @@ else
 end
 
 task :update do
-  tasks = [
-    "update:locales_resources",
-    "update:tailoring_data",
-    "update:unicode_data",
-    "update:unicode_properties",
-    "update:generate_casefolder",  # must come after unicode data
-    "update:composition_exclusions",
-    "update:postal_codes",
-    "update:phone_codes",
-    "update:language_codes",
-    "update:collation_tries",
-    "update:canonical_compositions",
-    "update:rbnf_tests",
-    "update:segment_exceptions",
-    "update:readme"
-  ]
+  tasks = if RUBY_PLATFORM == 'java'
+    # these should be run using JRuby 1.7 in 1.9 mode and CLDR v23 (v24 collation rules syntax is not supported yet)
+    [
+      "update:tailoring_data",  # per locale
+      "update:collation_tries", # per locale, must come after update:tailoring_data
+      "update:rbnf_tests",      # per locale
+    ]
+  else
+    puts "You might also want to run this rake task using JRuby 1.7 (in 1.9 mode) to update collation data and RBNF tests."
+
+    [
+      "update:locales_resources",      # per locale
+      "update:unicode_data",
+      "update:unicode_properties",
+      "update:generate_casefolder",    # must come after unicode data
+      "update:composition_exclusions",
+      "update:postal_codes",
+      "update:phone_codes",
+      "update:language_codes",
+      "update:canonical_compositions",
+      "update:segment_exceptions",     # per locale
+      "update:readme"
+    ]
+  end
 
   tasks.each do |task|
     puts "Executing #{task}"
@@ -85,6 +93,8 @@ task :update do
   end
 end
 
+# TODO: 'add_locale' task that creates a new directory and runs all necessary 'update' tasks (+ suggests to run those that depend on JRuby)
+
 namespace :update do
   ICU_JAR = './vendor/icu4j-51_2.jar'
 

data/lib/twitter_cldr/data_readers/number_data_reader.rb

@@ -11,8 +11,8 @@ module TwitterCldr
       ABBREVIATED_MIN_POWER = 3
       ABBREVIATED_MAX_POWER = 14
 
-      NUMBER_MAX = 10 ** (ABBREVIATED_MAX_POWER + 1)
       NUMBER_MIN = 10 ** ABBREVIATED_MIN_POWER
+      NUMBER_MAX = 10 ** (ABBREVIATED_MAX_POWER + 1)
 
       BASE_PATH   = [:numbers, :formats]
       SYMBOL_PATH = [:numbers, :symbols]
@@ -34,6 +34,10 @@ module TwitterCldr
 
       attr_reader :type, :format
 
+      def self.types
+        TYPE_PATHS.keys
+      end
+
       def initialize(locale, options = {})
         super(locale)
         @type = options[:type] || DEFAULT_TYPE
@@ -98,7 +102,7 @@ module TwitterCldr
 
       def valid_type_for?(number, type)
         if abbreviated?(type)
-          (number < NUMBER_MAX) && (number >= NUMBER_MIN)
+          self.class.within_abbreviation_range?(number)
         else
           TYPE_PATHS.include?(type)
         end
@@ -137,6 +141,9 @@ module TwitterCldr
         end
       end
 
+      def self.within_abbreviation_range?(number)
+        NUMBER_MIN <= number && number < NUMBER_MAX
+      end
     end
   end
 end

data/lib/twitter_cldr/data_readers/timespan_data_reader.rb

@@ -82,16 +82,8 @@ module TwitterCldr
         TwitterCldr::Formatters::TimespanFormatter.new(self)
       end
 
-      def all_types
-        traverse(BASE_PATH + [PATHS[direction][unit]]).keys
-      end
-
       protected
 
-      def default_type
-        all_types.first
-      end
-
       def pluralization_for(rule, available)
         # sometimes the plural rule will return ":one" when the resource only contains a path with "1"
         case rule
@@ -111,12 +103,19 @@ module TwitterCldr
       end
 
       def resource
-        @resource = begin
-          raw = TwitterCldr.get_locale_resource(locale, :units)
-          raw[TwitterCldr.convert_locale(locale)]
-        end
+        @resource ||= self.class.resource_for_locale(locale)
       end
 
+      class << self
+        def all_types_for(locale, unit, direction)
+          TwitterCldr::Utils.traverse_hash(resource_for_locale(locale), BASE_PATH + [PATHS[direction][unit]]).keys
+        end
+
+        def resource_for_locale(locale)
+          @resources ||= {}
+          @resources[locale] ||= TwitterCldr.get_locale_resource(locale, :units)[TwitterCldr.convert_locale(locale)]
+        end
+      end
     end
   end
 end

data/lib/twitter_cldr/formatters/numbers/abbreviated/abbreviated_number_formatter.rb

@@ -7,16 +7,18 @@ module TwitterCldr
   module Formatters
     class AbbreviatedNumberFormatter < NumberFormatter
 
-      protected
+      FORMAT_REGEX = /^0+$/
 
-      def transform_number(number)
-        within = number < TwitterCldr::DataReaders::NumberDataReader::NUMBER_MAX &&
-          number >= TwitterCldr::DataReaders::NumberDataReader::NUMBER_MIN
+      protected
 
-        if within
-          power = (((number.to_s.length - 1) / 3) * 3).floor
-          factor = (10 ** power).to_f
-          number / factor
+      def truncate_number(number, integer_format)
+        if TwitterCldr::DataReaders::NumberDataReader.within_abbreviation_range?(number)
+          if integer_format.format =~ FORMAT_REGEX
+            factor = [0, number.to_i.to_s.length - integer_format.format.length].max
+            number / (10.0 ** factor)
+          else
+            raise ArgumentError.new("unexpected format string #{integer_format.inspect}")
+          end
         else
           number
         end

data/lib/twitter_cldr/formatters/numbers/number_formatter.rb

@@ -25,7 +25,7 @@ module TwitterCldr
         options[:type] ||= :decimal
 
         prefix, suffix, integer_format, fraction_format = *partition_tokens(tokens)
-        number = transform_number(number)
+        number = truncate_number(number, integer_format)
 
         int, fraction = parse_number(number, options)
         result =  integer_format.apply(int, options)
@@ -46,7 +46,7 @@ module TwitterCldr
         )
       end
 
-      def transform_number(number)
+      def truncate_number(number, integer_format)
         number  # noop for base class
       end
 

data/lib/twitter_cldr/localized/localized_datetime.rb

@@ -13,7 +13,7 @@ module TwitterCldr
 
       class << self
         def types
-          TwitterCldr::DataReaders::CalendarDataReader::TYPE_PATHS.keys
+          TwitterCldr::DataReaders::CalendarDataReader.types
         end
       end
 

data/lib/twitter_cldr/localized/localized_number.rb

@@ -17,7 +17,7 @@ module TwitterCldr
 
       class << self
         def types
-          TwitterCldr::DataReaders::NumberDataReader::TYPE_PATHS.keys
+          TwitterCldr::DataReaders::NumberDataReader.types
         end
       end
 
@@ -33,8 +33,18 @@ module TwitterCldr
           :format => @format
         })
 
-        tokens = data_reader.tokenizer.tokenize(data_reader.pattern(base_obj))
-        data_reader.formatter.format(tokens, base_obj, options.merge(:type => @type))
+        pattern = data_reader.pattern(base_obj)
+
+        if pattern == 0
+          # can't format the number for current locale
+          base_obj.to_s
+        else
+          data_reader.formatter.format(
+            data_reader.tokenizer.tokenize(pattern),
+            base_obj,
+            options.merge(:type => @type)
+          )
+        end
       end
 
       def plural_rule

data/lib/twitter_cldr/resources/collation_tries_dumper.rb

@@ -12,8 +12,7 @@ module TwitterCldr
 
         def update_dumps
           update_default_trie_dump
-          #TwitterCldr.supported_locales
-          [:he].each { |locale| update_tailoring_trie_dump(locale) }
+          TwitterCldr.supported_locales.each { |locale| update_tailoring_trie_dump(locale) }
         end
 
         private

data/lib/twitter_cldr/resources/language_codes_importer.rb

@@ -13,7 +13,7 @@ module TwitterCldr
       BCP_47_FILE, ISO_639_FILE = %w[bcp-47.txt iso-639.txt]
 
       INPUT_DATA = {
-        BCP_47_FILE  => 'http://www.iana.org/assignments/language-subtag-registry',
+        BCP_47_FILE  => 'http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry',
         ISO_639_FILE => 'http://www-01.sil.org/iso639-3/iso-639-3_20120614.tab'
       }
 

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -3,6 +3,7 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
+require 'fileutils'
 require 'cldr/export'
 
 require 'twitter_cldr/resources/download'
@@ -12,7 +13,8 @@ module TwitterCldr
 
     class LocalesResourcesImporter
 
-      # NOTE: units.yml was NOT updated to cldr 24 (too many significant changes) - add back in when appropriate
+      # NOTE: units.yml was NOT updated to cldr 24 (too many significant changes) - add back in when appropriate.
+      #       Meanwhile, use ruby-cldr v0.0.2 and CLDR 22.1 to update units.yml files.
       LOCALE_COMPONENTS = %w[calendars languages numbers plurals lists layout currencies territories rbnf]  # units
       SHARED_COMPONENTS = %w[currency_digits_and_rounding rbnf_root numbering_systems segments_root]
 
@@ -43,7 +45,15 @@ module TwitterCldr
       def copy_zh_hant_plurals
         File.open(File.join(@output_path, 'locales', 'zh-Hant', 'plurals.yml'), 'w:utf-8') do |output|
           data = YAML.load(File.read(File.join(@output_path, 'locales', 'zh', 'plurals.yml')))
-          output.write(YAML.dump(:'zh-Hant' => data[:zh].gsub(':zh', ":'zh-Hant'")))
+          output.write(YAML.dump(:'zh-Hant' => data[:zh].gsub(":'zh'", ":'zh-Hant'")))
+        end
+      end
+
+      def move_segments_root_file
+        file_path = File.join(@output_path, 'shared', 'segments_root.yml')
+
+        if File.file?(file_path)
+          FileUtils.move(file_path, File.join(@output_path, 'shared', 'segments', 'segments_root.yml'))
         end
       end
 
@@ -72,6 +82,7 @@ module TwitterCldr
           deep_symbolize(component, locale, path)
         end
 
+        move_segments_root_file
         copy_zh_hant_plurals
       end
 

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -10,7 +10,8 @@ require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources
-    # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version >= 49.1.
+    # This class should be used with JRuby 1.7 in 1.9 mode, ICU4J version >= 49.1,
+    # and CLDR version <= 23 (v24 syntax is not supported yet).
     #
     class TailoringImporter < IcuBasedImporter
 
@@ -225,7 +226,7 @@ module TwitterCldr
       end
 
       def get_code_points(string)
-        TwitterCldr::Normalization::NFD.normalize_code_points(TwitterCldr::Utils::CodePoints.from_string(string))
+        TwitterCldr::Utils::CodePoints.from_string(TwitterCldr::Normalization.normalize(string))
       end
 
     end

data/lib/twitter_cldr/shared/calendar.rb

@@ -86,6 +86,10 @@ module TwitterCldr
         get_order_for(TwitterCldr::DataReaders::DateTimeDataReader, options)
       end
 
+      def calendar_data
+        @calendar_data ||= TwitterCldr::Utils.traverse_hash(resource, [locale, :calendars, calendar_type])
+      end
+
       private
 
       def cache_field_data(field, names_form = nil, format = nil)
@@ -154,10 +158,6 @@ module TwitterCldr
         Regexp.new("^calendars\.#{calendar_type}\.(.*)$")
       end
 
-      def calendar_data
-        @calendar_data ||= TwitterCldr::Utils.traverse_hash(resource, [locale, :calendars, calendar_type])
-      end
-
       def resource
         TwitterCldr.get_locale_resource(@locale, :calendars)
       end

data/lib/twitter_cldr/shared/postal_codes.rb

@@ -19,13 +19,11 @@ module TwitterCldr
         def for_territory(territory)
           key = territory.to_s.downcase.to_sym
           if res = resource[key]
-            territory_cache[key] ||= begin
-              new(
-                territory,
-                res[:regex],
-                TwitterCldr::Utils::RegexpAst.load(res[:ast])
-              )
-            end
+            territory_cache[key] ||= new(
+              territory,
+              res[:regex],
+              TwitterCldr::Utils::RegexpAst.load(res[:ast])
+            )
           else
             raise InvalidTerritoryError, "invalid territory"
           end

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "3.0.3"
+  VERSION = "3.0.4"
 end

data/resources/collation/tailoring/de-CH.yml

@@ -0,0 +1,4 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''

data/resources/collation/tailoring/en-AU.yml

@@ -0,0 +1,4 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''

data/resources/collation/tailoring/en-CA.yml

@@ -0,0 +1,4 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''

data/resources/collation/tailoring/en-IE.yml

@@ -0,0 +1,4 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''