twitter_cldr 1.6.2 → 1.7.0

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -39,12 +39,12 @@ module TwitterCldr
         end
 
         def parse_code_points(string)
-          string.split.map { |cp| cp.to_i(16) }
+          string.split.map(&:hex)
         end
 
         def parse_collation_element(string)
           string.scan(/\[.*?\]/).map do |match|
-            match[1..-2].gsub(/\s/, '').split(',', -1).map { |bytes| bytes.to_i(16) }
+            match[1..-2].gsub(/\s/, '').split(',', -1).map { |bytes| bytes.hex }
           end
         end
 
@@ -71,7 +71,7 @@ module TwitterCldr
 
         def parse_suppressed_starters(suppressed_contractions)
           suppressed_contractions.chars.map do |starter|
-            TwitterCldr::Utils::CodePoints.from_string(starter).first.to_i(16)
+            TwitterCldr::Utils::CodePoints.from_string(starter).first
           end
         end
 

data/lib/twitter_cldr/formatters/numbers/decimal_formatter.rb

@@ -20,12 +20,6 @@ module TwitterCldr
       def default_format_options_for(number)
         { :precision => precision_from(number) }
       end
-
-      protected
-
-      def get_tokens(obj, options = {})
-        obj.abs == obj ? @tokenizer.tokens(:sign => :positive) : @tokenizer.tokens(:sign => :negative)
-      end
     end
   end
 end

data/lib/twitter_cldr/formatters/numbers/helpers/base.rb

@@ -10,7 +10,7 @@ module TwitterCldr
         def interpolate(string, value, orientation = :right)
           value  = value.to_s
           length = value.length
-          start, pad = orientation == :left ? [0, :rjust] : [-length, :ljust]
+          start = orientation == :left ? 0 : -length
 
           string = string.dup
           string = string.ljust(length, '#') if string.length < length

data/lib/twitter_cldr/formatters/numbers/number_formatter.rb

@@ -35,8 +35,8 @@ module TwitterCldr
 
       def parse_number(number, options = {})
         precision = options[:precision] || self.precision_from(number)
-        number = round_to(number, precision)
-        number.abs.to_s.split(".")
+        number = "%.#{precision}f" % round_to(number, precision).abs
+        number.split(".")
       end
 
       def round_to(number, precision)
@@ -48,6 +48,10 @@ module TwitterCldr
         parts = num.to_s.split(".")
         parts.size == 2 ? parts[1].size : 0
       end
+
+      def get_tokens(obj, options = {})
+        obj.abs == obj ? @tokenizer.tokens(:sign => :positive) : @tokenizer.tokens(:sign => :negative)
+      end
     end
   end
 end

data/lib/twitter_cldr/normalization/base.rb

@@ -9,8 +9,14 @@ module TwitterCldr
 
       class << self
 
+        def normalize(string)
+          code_points = TwitterCldr::Utils::CodePoints.from_string(string)
+          normalized_code_points = normalize_code_points(code_points)
+          TwitterCldr::Utils::CodePoints.to_string(normalized_code_points)
+        end
+
         def combining_class_for(code_point)
-          TwitterCldr::Shared::CodePoint.for_hex(code_point).combining_class.to_i
+          TwitterCldr::Shared::CodePoint.find(code_point).combining_class.to_i
         rescue NoMethodError
           0
         end

data/lib/twitter_cldr/normalization/nfd.rb

@@ -15,12 +15,8 @@ module TwitterCldr
 
         protected
 
-        # Returns code point's Decomposition Mapping based on its Unicode data. Returns nil if the mapping has
-        # compatibility type (it contains compatibility formatting tag).
-        #
-        def decomposition_mapping(unicode_data)
-          mapping = parse_decomposition_mapping(unicode_data)
-          mapping unless compatibility_decomposition?(mapping)
+        def decompose?(unicode_data)
+          super && !unicode_data.compatibility_decomposition? # skip compatibility decompositions
         end
 
       end

data/lib/twitter_cldr/normalization/nfkc.rb

@@ -13,12 +13,6 @@ module TwitterCldr
 
       class << self
 
-        def normalize(string)
-          code_points = TwitterCldr::Utils::CodePoints.from_string(string)
-          normalized_code_points = normalize_code_points(code_points)
-          TwitterCldr::Utils::CodePoints.to_string(normalized_code_points)
-        end
-
         def normalize_code_points(code_points)
           compose(TwitterCldr::Normalization::NFKD.normalize_code_points(code_points))
         end
@@ -59,7 +53,7 @@ module TwitterCldr
         end
 
         def compose_hangul(code_points)
-          TwitterCldr::Normalization::Hangul.compose(code_points.map { |cp| cp.hex }).to_s(16).upcase.rjust(4, "0")
+          TwitterCldr::Normalization::Hangul.compose(code_points)
         end
 
         # Implements composition of Unicode code points following the guidelines here:
@@ -86,12 +80,12 @@ module TwitterCldr
 
             unless blocked
               # do a reverse-lookup for the decomposed code points
-              decomp_data = TwitterCldr::Shared::CodePoint.for_decomposition([code_points[starter_index], code_point])
+              composite = TwitterCldr::Shared::CodePoint.for_canonical_decomposition([code_points[starter_index], code_point])
 
               # check if two code points are canonically equivalent
-              if decomp_data && !decomp_data.excluded_from_composition?
+              if composite && !composite.excluded_from_composition?
                 # combine the characters
-                code_points[starter_index] = decomp_data.code_point
+                code_points[starter_index] = composite.code_point
                 code_points.delete_at(index)
                 index -= 1
               end

data/lib/twitter_cldr/normalization/nfkd.rb

@@ -16,65 +16,48 @@ module TwitterCldr
     #
     class NFKD < Base
 
-      class << self
 
-        def normalize(string)
-          code_points = TwitterCldr::Utils::CodePoints.from_string(string)
-          normalized_code_points = normalize_code_points(code_points)
-          TwitterCldr::Utils::CodePoints.to_string(normalized_code_points)
-        end
+      class << self
 
         def normalize_code_points(code_points)
-          canonical_ordering(decomposition(code_points))
+          canonical_ordering(decompose(code_points))
         end
 
         protected
 
-        def decomposition(code_points)
+        def decompose(code_points)
           code_points.map { |code_point| decompose_recursively(code_point) }.flatten
         end
 
         # Recursively decomposes a given code point with the values in its Decomposition Mapping property.
         #
         def decompose_recursively(code_point)
-          unicode_data = TwitterCldr::Shared::CodePoint.for_hex(code_point)
+          unicode_data = TwitterCldr::Shared::CodePoint.find(code_point)
           return code_point unless unicode_data
 
           if unicode_data.hangul_type == :compositions
             decompose_hangul(code_point)
           else
-            decompose_regular(code_point, decomposition_mapping(unicode_data))
+            decompose_regular(unicode_data)
           end
         end
 
         # Decomposes regular (non-Hangul) code point.
         #
-        def decompose_regular(code_point, mapping)
-          if mapping && !mapping.empty?
-            mapping.map{ |cp| decompose_recursively(cp) }.flatten
+        def decompose_regular(unicode_data)
+          if decompose?(unicode_data)
+            unicode_data.decomposition.map { |code_point| decompose_recursively(code_point) }.flatten
           else
-            code_point
+            unicode_data.code_point
           end
         end
 
-        # Returns code point's Decomposition Mapping based on its Unicode data.
-        #
-        def decomposition_mapping(unicode_data)
-          mapping = parse_decomposition_mapping(unicode_data)
-          mapping.shift if compatibility_decomposition?(mapping) # remove compatibility formatting tag
-          mapping
-        end
-
-        def compatibility_decomposition?(mapping)
-          !!(COMPATIBILITY_FORMATTING_TAG_REGEXP =~ mapping.first)
-        end
-
-        def parse_decomposition_mapping(unicode_data)
-          unicode_data.decomposition.split
+        def decompose?(unicode_data)
+          !!unicode_data.decomposition
         end
 
         def decompose_hangul(code_point)
-          TwitterCldr::Normalization::Hangul.decompose(code_point.hex).map { |e| e.to_s(16).upcase }
+          TwitterCldr::Normalization::Hangul.decompose(code_point)
         end
 
         # Performs the Canonical Ordering Algorithm by stable sorting of every subsequence of combining code points
@@ -127,16 +110,8 @@ module TwitterCldr
           code_points_with_cc
         end
 
-        def combining_class_for(code_point)
-          TwitterCldr::Shared::CodePoint.for_hex(code_point).combining_class.to_i
-        rescue NoMethodError
-          0
-        end
-
       end
 
-      COMPATIBILITY_FORMATTING_TAG_REGEXP = /^<.*>$/
-
     end
   end
 end

data/lib/twitter_cldr/resources.rb

@@ -5,8 +5,15 @@
 
 module TwitterCldr
   module Resources
-    autoload :Loader,            'twitter_cldr/resources/loader'
-    autoload :TailoringImporter, 'twitter_cldr/resources/tailoring_importer'
-    autoload :TriesDumper,       'twitter_cldr/resources/tries_dumper'
+    autoload :CanonicalCompositionsUpdater,   'twitter_cldr/resources/canonical_compositions_updater'
+    autoload :CompositionExclusionsImporter,  'twitter_cldr/resources/composition_exclusions_importer'
+    autoload :CustomLocalesResourcesImporter, 'twitter_cldr/resources/custom_locales_resources_importer'
+    autoload :Loader,                         'twitter_cldr/resources/loader'
+    autoload :LocalesResourcesImporter,       'twitter_cldr/resources/locales_resources_importer'
+    autoload :PhoneCodesImporter,             'twitter_cldr/resources/phone_codes_importer'
+    autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
+    autoload :TailoringImporter,              'twitter_cldr/resources/tailoring_importer'
+    autoload :TriesDumper,                    'twitter_cldr/resources/tries_dumper'
+    autoload :UnicodeDataImporter,            'twitter_cldr/resources/unicode_data_importer'
   end
 end

data/lib/twitter_cldr/resources/canonical_compositions_updater.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+
+    class CanonicalCompositionsUpdater
+
+      CODE_POINT_MAX = 0x10FFFF
+
+      # Arguments:
+      #
+      #   output_path - output directory for generated YAML file
+      #
+      def initialize(output_path)
+        @output_path = output_path
+      end
+
+      def update
+        File.open(File.join(@output_path, 'canonical_compositions.yml'), 'w') do |output|
+          YAML.dump(generate_compositions, output)
+        end
+      end
+
+      private
+
+      def generate_compositions
+        (1..CODE_POINT_MAX).inject({}) do |memo, code_point|
+          code_point_data = TwitterCldr::Shared::CodePoint.find(code_point)
+
+          if code_point_data && !code_point_data.compatibility_decomposition? && code_point_data.decomposition && !code_point_data.decomposition.empty?
+            memo[code_point_data.decomposition] = code_point
+          end
+
+          log_progress(code_point, memo.size)
+
+          memo
+        end
+      end
+
+      def log_progress(code_point, compositions_count)
+        $stdout.write("\r#{(100.0 * code_point / CODE_POINT_MAX).round}% complete, found #{compositions_count} canonical compositions")
+        $stdout.write("\n") if code_point == CODE_POINT_MAX
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/composition_exclusions_importer.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'lib/twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class CompositionExclusionsImporter
+
+      COMPOSITION_EXCLUSIONS_URL   = 'http://www.unicode.org/Public/6.1.0/ucd/DerivedNormalizationProps.txt'
+      COMPOSITION_EXCLUSION_REGEXP = /^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+; Full_Composition_Exclusion #.*$/
+      TOTAL_CODE_POINTS_REGEXP     = /^# Total code points: (\d+)$/
+
+      # Arguments:
+      #
+      #   input_path  - path to DerivedNormalizationProps.txt file
+      #   output_path - output directory for generated YAML file
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        File.open(File.join(@output_path, 'composition_exclusions.yml'), 'w') do |output|
+          YAML.dump(generate_composition_exclusions, output)
+        end
+      end
+
+      private
+
+      def generate_composition_exclusions
+        data      = File.open(composition_exclusions_file) { |file| file.read }
+        start_pos = data.index("# Derived Property: Full_Composition_Exclusion")
+        end_pos   = data.index(/^#\s=*$/, start_pos)
+        data      = data[start_pos..end_pos].split("\n")
+
+        expected_code_points_count = nil
+
+        result = data.inject([]) do |memo, line|
+          memo << ($1.hex..($2 || $1).hex)     if line =~ COMPOSITION_EXCLUSION_REGEXP
+          expected_code_points_count = $1.to_i if line =~ TOTAL_CODE_POINTS_REGEXP
+          memo
+        end
+
+        raise "Expected number of code points was not found." unless expected_code_points_count
+        code_points_count = result.map(&:count).inject(:+)
+        raise "Unexpected number of code points: expected - #{expected_code_points_count}, got - #{code_points_count}." unless code_points_count == expected_code_points_count
+
+        result
+      end
+
+      def composition_exclusions_file
+        TwitterCldr::Resources.download_if_necessary(@input_path, COMPOSITION_EXCLUSIONS_URL)
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/custom_locales_resources_importer.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'json'
+require 'open-uri'
+
+module TwitterCldr
+  module Resources
+
+    class CustomLocalesResourcesImporter
+
+      API_ENDPOINT = "http://translate.twttr.com/api/2/twitter/phrase/%s/translations.json"
+
+      TIME_PERIODS = {
+          :day    => 19636,
+          :hour   => 19638,
+          :minute => 19634,
+          :second => 19639
+      }
+
+      # Arguments:
+      #
+      #   output_path - output directory for imported YAML files
+      #
+      def initialize(output_path)
+        @output_path = output_path
+      end
+
+      def import
+        import_units
+      end
+
+      private
+
+      def import_units
+        fetch_units_data.each do |locale, data|
+          dir_path = File.join(@output_path, locale.to_s)
+
+          FileUtils.mkpath(dir_path)
+
+          File.open(File.join(dir_path, 'units.yml'), 'w:utf-8') do |output|
+            output.write(YAML.dump({ locale => data }))
+          end
+        end
+      end
+
+      def fetch_units_data
+        TIME_PERIODS.inject({}) do |result, (label, id)|
+          api_response = JSON.parse(open(API_ENDPOINT % id).read)
+
+          TwitterCldr.supported_locales.each do |locale|
+            twitter_locale = TwitterCldr.twitter_locale(locale).to_s
+
+            next unless api_response[twitter_locale]
+
+            patterns = TwitterCldr::Formatters::Plurals::Rules.all_for(locale).inject({}) do |memo, rule|
+              memo[rule] = api_response[twitter_locale].gsub("%{number}", "{0}"); memo
+            end
+
+            set_value(result, patterns, locale, :units, label, :abbreviated)
+          end
+
+          result
+        end
+      end
+
+      def set_value(hash, value, *path)
+        last = path[0..-2].inject(hash) do |current, level|
+          current[level] ||= {}
+        end
+
+        last[path.last] = value
+      end
+
+    end
+
+  end
+end