twitter_cldr 1.5.0 → 1.6.0

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -6,48 +6,74 @@
 module TwitterCldr
   module Collation
 
-    # Builds a collation elements Trie from the file containing a fractional collation elements table.
+    # Builds a fractional collation elements Trie from the file containing a fractional collation elements table.
     #
-    class TrieBuilder
+    module TrieBuilder
 
-      FRACTIONAL_UCA_REGEXP = /^((?:[0-9A-F]+)(?:\s[0-9A-F]+)*);\s((?:\[.*?\])(?:\[.*?\])*)/
+      # Fractional collation element regexp
+      FCE_REGEXP = /^((?:[0-9A-F]+)(?:\s[0-9A-F]+)*);\s((?:\[.*?\])(?:\[.*?\])*)/
 
-      def self.load_trie(file_path)
-        new(file_path).build
-      end
+      class << self
 
-      def initialize(resource)
-        @file_path = File.join(TwitterCldr::RESOURCES_DIR, resource)
-      end
+        def load_trie(resource)
+          parse_trie(load_resource(resource))
+        end
 
-      def build
-        parse_trie(load_collation_elements_table)
-      end
+        def load_tailored_trie(locale, fallback)
+          build_tailored_trie(TwitterCldr.get_resource(:collation, :tailoring, locale), fallback)
+        end
 
-      private
+        def parse_trie(table, trie = TwitterCldr::Collation::Trie.new)
+          table.lines.each do |line|
+            trie.set(parse_code_points($1), parse_collation_element($2)) if FCE_REGEXP =~ line
+          end
 
-      def parse_trie(table)
-        trie = TwitterCldr::Collation::Trie.new
+          trie
+        end
+
+        private
 
-        table.lines.each do |line|
-          trie.add(parse_code_points($1), parse_collation_element($2)) if FRACTIONAL_UCA_REGEXP =~ line
+        def load_resource(resource)
+          open(File.join(TwitterCldr::RESOURCES_DIR, resource), 'r')
         end
 
-        trie
-      end
+        def parse_code_points(string)
+          string.split.map { |cp| cp.to_i(16) }
+        end
 
-      def load_collation_elements_table
-        open(@file_path, 'r')
-      end
+        def parse_collation_element(string)
+          string.scan(/\[.*?\]/).map do |match|
+            match[1..-2].gsub(/\s/, '').split(',', -1).map { |bytes| bytes.to_i(16) }
+          end
+        end
 
-      def parse_code_points(string)
-        string.split.map { |cp| cp.to_i(16) }
-      end
+        def build_tailored_trie(tailoring_data, fallback)
+          trie = TwitterCldr::Collation::TrieWithFallback.new(fallback)
+
+          parse_trie(tailoring_data[:tailored_table], trie)
+          copy_expansions(trie, fallback, parse_suppressed_starters(tailoring_data[:suppressed_contractions]))
 
-      def parse_collation_element(string)
-        string.scan(/\[.*?\]/).map do |match|
-          match[1..-2].gsub(/\s/, '').split(',', -1).map { |bytes| bytes.to_i(16) }
+          trie
         end
+
+        def copy_expansions(trie, source_trie, suppressed_starters)
+          suppressed_starters.each do |starter|
+            trie.add([starter], source_trie.get([starter]))
+          end
+
+          (trie.starters - suppressed_starters).each do |starter|
+            source_trie.each_starting_with(starter) do |key, value|
+              trie.add(key, value)
+            end
+          end
+        end
+
+        def parse_suppressed_starters(suppressed_contractions)
+          suppressed_contractions.chars.map do |starter|
+            TwitterCldr::Utils::CodePoints.from_string(starter).first.to_i(16)
+          end
+        end
+
       end
 
     end

data/lib/twitter_cldr/collation/trie_with_fallback.rb

@@ -0,0 +1,55 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Collation
+
+    # Trie that delegates all not found keys to the fallback.
+    #
+    # Note: methods #get and #find_prefix have a bit different behavior. The first one, #get, delegates to the fallback
+    # any key that was not found. On the other hand, #find_refix delegates the key only if none of its prefixes was
+    # found.
+    #
+    # E.g., if the fallback contains key [1, 2] with value '12' and the trie itself contains only key [1] with value '1'
+    # results will be the following:
+    #
+    #   trie.get([1, 2]) #=> '12' - key [1, 2] wasn't found in the trie, so it was delegated to the fallback where the
+    #                               value '12' was found.
+    #
+    #   trie.find_prefix([1, 2]) #=> ['1', 1, suffixes] - key [1, 2] is not present in the trie, but its prefix [1] was
+    #                                                     found, so the fallback wasn't used.
+    #
+    #   trie.find_prefix([3, 2]) - the trie itself includes neither key [3, 2] nor its prefix [3], so this call is
+    #                              delegated to the fallback.
+    #
+    # This special behavior of the #find_prefix method allows 'hiding' fallback keys that contain more than one element
+    # by adding their one element prefixes to the trie itself. This feature is useful for some applications, e.g., for
+    # suppressing contractions in a tailored FCE trie.
+    #
+    class TrieWithFallback < TwitterCldr::Collation::Trie
+
+      def initialize(fallback)
+        super()
+        @fallback = fallback
+      end
+
+      def get(key)
+        super || @fallback.get(key)
+      end
+
+      def find_prefix(key)
+        value, prefix_size, suffixes = super
+
+        if prefix_size > 0
+          [value, prefix_size, suffixes]
+        else
+          @fallback.find_prefix(key)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/core_ext/array.rb

@@ -12,11 +12,24 @@ end
 module TwitterCldr
   class LocalizedArray < LocalizedObject
     def code_points_to_string
-      TwitterCldr::Utils::CodePoints.to_string(self.base_obj)
+      TwitterCldr::Utils::CodePoints.to_string(base_obj)
+    end
+
+    def sort
+      TwitterCldr::Collation::Collator.new(locale).sort(base_obj).localize
+    end
+
+    def sort!
+      TwitterCldr::Collation::Collator.new(locale).sort!(base_obj)
+      self
     end
 
     def formatter_const
       nil
     end
+
+    def to_a
+      @base_obj.dup
+    end
   end
 end

data/lib/twitter_cldr/core_ext/calendars/datetime.rb

@@ -24,18 +24,24 @@ module TwitterCldr
       end
     end
 
+    def to_timespan(options = {})
+      base_time = options[:base_time] || Time.now
+      seconds = (self.to_time.base_obj.to_i - base_time.to_i).abs
+      TwitterCldr::LocalizedTimespan.new(seconds, options.merge(:locale => @locale, :direction => :none))
+    end
+
     def ago(options = {})
       base_time = options[:base_time] || Time.now
       seconds = self.to_time.base_obj.to_i - base_time.to_i
       raise ArgumentError.new('Start date is after end date. Consider using "until" function.') if seconds > 0
-      TwitterCldr::LocalizedTimespan.new(seconds, @locale).to_s(options[:unit])
+      TwitterCldr::LocalizedTimespan.new(seconds, options.merge(:locale => @locale))
     end
 
     def until(options = {})
       base_time = options[:base_time] || Time.now
       seconds = self.to_time.base_obj.to_i - base_time.to_i
       raise ArgumentError.new('End date is before start date. Consider using "ago" function.') if seconds < 0
-      TwitterCldr::LocalizedTimespan.new(seconds, @locale).to_s(options[:unit])
+      TwitterCldr::LocalizedTimespan.new(seconds, options.merge(:locale => @locale))
     end
 
     def to_s

data/lib/twitter_cldr/core_ext/calendars/timespan.rb

@@ -6,13 +6,13 @@
 module TwitterCldr
   class LocalizedTimespan < LocalizedObject
 
-    def initialize(seconds, locale)
-      @formatter = TwitterCldr::Formatters::TimespanFormatter.new(:locale => locale)
-      @seconds = seconds
+    def initialize(seconds, options = {})
+      super(seconds, options[:locale], options)
+      @formatter = TwitterCldr::Formatters::TimespanFormatter.new(options)
     end
 
-    def to_s(unit = :default)
-      @formatter.format(@seconds, unit)
+    def to_s(options = {})
+      @formatter.format(@base_obj, options)
     end
 
     protected

data/lib/twitter_cldr/formatters/calendars/timespan_formatter.rb

@@ -6,6 +6,8 @@
 module TwitterCldr
   module Formatters
     class TimespanFormatter < Base
+      DEFAULT_TYPE = :default
+
       TIME_IN_SECONDS = { 
         :second => 1,
         :minute => 60,
@@ -17,21 +19,19 @@ module TwitterCldr
       }
 
       def initialize(options = {})
+        @direction = options[:direction]
         @tokenizer = TwitterCldr::Tokenizers::TimespanTokenizer.new(:locale => extract_locale(options))
       end
 
-      def format(seconds, unit)
-        direction = seconds < 0 ? :ago : :until
-
-        if unit.nil? || unit == :default
-          unit = self.calculate_unit(seconds.abs)
-        end
-
-        number = calculate_time(seconds.abs, unit)
+      def format(seconds, options = {})
+        options[:direction] ||= @direction || (seconds < 0 ? :ago : :until)
+        options[:unit] ||= self.calculate_unit(seconds.abs)
+        options[:number] = calculate_time(seconds.abs, options[:unit])
+        options[:type] ||= DEFAULT_TYPE
 
-        tokens = @tokenizer.tokens(:direction => direction, :unit => unit, :number => number)
+        tokens = @tokenizer.tokens(options)
         strings = tokens.map { |token| token[:value]}
-        strings.join.gsub(/\{[0-9]\}/, number.to_s)
+        strings.join.gsub(/\{[0-9]\}/, options[:number].to_s)
       end
 
       def calculate_unit(seconds)

data/lib/twitter_cldr/formatters/plurals/rules.rb

@@ -15,15 +15,13 @@ module TwitterCldr
           end
 
           def all_for(locale)
-            locale = TwitterCldr.convert_locale(locale.to_sym)
-            get_resource(locale)[locale][:i18n][:plural][:keys]
+            get_resource(locale)[:keys]
           rescue
             nil
           end
 
           def rule_for(number, locale = TwitterCldr.get_locale)
-            locale = TwitterCldr.convert_locale(locale.to_sym)
-            get_resource(locale)[locale][:i18n][:plural][:rule].call(number)
+            get_resource(locale)[:rule].call(number)
           rescue
             :other
           end
@@ -32,7 +30,7 @@ module TwitterCldr
 
           def get_resource(locale)
             locale = TwitterCldr.convert_locale(locale)
-            eval(TwitterCldr.get_locale_resource(locale, :plurals)[locale])
+            eval(TwitterCldr.get_locale_resource(locale, :plurals)[locale])[locale][:i18n][:plural]
           end
 
         end

data/lib/twitter_cldr/resources.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    autoload :Loader, 'twitter_cldr/resources/loader'
+    autoload :Import, 'twitter_cldr/resources/import'
+  end
+end

data/lib/twitter_cldr/resources/import.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Import
+      autoload :Tailoring, 'twitter_cldr/resources/import/tailoring'
+    end
+  end
+end

data/lib/twitter_cldr/resources/import/tailoring.rb

@@ -0,0 +1,193 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'yaml'
+require 'java'
+
+module TwitterCldr
+  module Resources
+    module Import
+
+      # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version 49.1 (available at
+      # http://download.icu-project.org/files/icu4j/49.1/icu4j-49_1.jar).
+      #
+      class Tailoring
+
+        SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
+        SIMPLE_RULES      = %w[p s t i]
+        LEVEL_RULE_REGEXP = /^(p|s|t|i)(c?)$/
+
+        IGNORED_TAGS = %w[reset text #comment]
+
+        LAST_BYTE_MASK = 0xFF
+
+        LOCALES_MAP = {
+            :'zh-Hant' => :'zh_Hant',
+            :id => :root,
+            :it => :root,
+            :ms => :root,
+            :nl => :root,
+            :pt => :root
+        }
+
+        EMPTY_TAILORING_DATA = { 'tailored_table' => '', 'suppressed_contractions' => '' }
+
+        class ImportError < RuntimeError; end
+
+        # Arguments:
+        #
+        #   input_path  - path to a directory containing CLDR tailoring data (available at
+        #                 http://unicode.org/cldr/trac/browser/tags/release-21/common/collation/
+        #                 or as a part of CLDR release at http://cldr.unicode.org/index/downloads)
+        #
+        #   output_path - output directory for imported YAML files
+        #
+        #   icu4j_path  - path to ICU4J jar file
+        #
+        def initialize(input_path, output_path, icu4j_path)
+          require icu4j_path
+
+          @input_path  = input_path
+          @output_path = output_path
+        end
+
+        def import(locale)
+          print "Importing %8s\t--\t" % locale
+
+          if tailoring_present?(locale)
+            YAML.dump(tailoring_data(locale), open(resource_file_path(locale), 'w'))
+            puts "Done."
+          else
+            YAML.dump(EMPTY_TAILORING_DATA, open(resource_file_path(locale), 'w'))
+            puts "Missing (generated empty tailoring resource)."
+          end
+        rescue ImportError => e
+          puts "Error: #{e.message}"
+        end
+
+        private
+
+        def tailoring_present?(locale)
+          File.file?(locale_file_path(locale))
+        end
+
+        def translated_locale(locale)
+          LOCALES_MAP.fetch(locale, locale)
+        end
+
+        def locale_file_path(locale)
+          File.join(@input_path, "#{translated_locale(locale)}.xml")
+        end
+
+        def resource_file_path(locale)
+          File.join(@output_path, "#{locale}.yml")
+        end
+
+        def tailoring_data(locale)
+          doc = Nokogiri::XML(open(locale_file_path(locale)))
+          collations = doc.at_xpath('//collations')
+
+          collation_alias = collations.at_xpath('alias[@path="//ldml/collations"]')
+          aliased_locale = collation_alias && collation_alias.attr('source')
+
+          return tailoring_data(aliased_locale) if aliased_locale
+
+          standard_tailoring = collations.at_xpath('collation[@type="standard"]')
+
+          {
+              'tailored_table'          => parse_tailorings(standard_tailoring, locale),
+              'suppressed_contractions' => parse_suppressed_contractions(standard_tailoring)
+          }
+        end
+
+        def parse_tailorings(data, locale)
+          rules = data && data.at_xpath('rules')
+
+          return '' unless rules
+
+          collator = Java::ComIbmIcuText::Collator.get_instance(Java::JavaUtil::Locale.new(locale.to_s))
+
+          rules.children.map do |child|
+            validate_tailoring_rule(child)
+
+            if child.name =~ LEVEL_RULE_REGEXP
+              if $2.empty?
+                table_entry_for_rule(collator, child.text)
+              else
+                child.text.chars.map { |char| table_entry_for_rule(collator, char) }
+              end
+            elsif child.name == 'x'
+              context = ''
+              child.children.each_with_object([]) do |c, memo|
+                if SIMPLE_RULES.include?(c.name)
+                  memo << table_entry_for_rule(collator, context + c.text)
+                elsif c.name == 'context'
+                  context = c.text
+                elsif c.name != 'extend'
+                  raise ImportError, "Rule '#{c.name}' inside <x></x> is not supported."
+                end
+              end
+            else
+              raise ImportError, "Tag '#{child.name}' is not supported." unless IGNORED_TAGS.include?(child.name)
+            end
+          end.flatten.compact.join("\n")
+        end
+
+        def table_entry_for_rule(collator, tailored_value)
+          code_points = get_code_points(tailored_value)
+
+          collation_elements = get_collation_elements(collator, tailored_value).map do |ce|
+            ce.map { |l| l.to_s(16).upcase }.join(', ')
+          end
+
+          "#{code_points.join(' ')}; [#{collation_elements.join('][')}]"
+        end
+
+        def parse_suppressed_contractions(data)
+          return '' unless data
+
+          Array(data.xpath('suppress_contractions')).map do |contractions|
+            Java::ComIbmIcuText::UnicodeSet.to_array(Java::ComIbmIcuText::UnicodeSet.new(contractions.text)).to_a
+          end.flatten.join
+        end
+
+        def validate_tailoring_rule(rule)
+          return if IGNORED_TAGS.include?(rule.name)
+
+          raise ImportError, "Rule '#{rule.name}' is not supported." unless SUPPORTED_RULES.include?(rule.name)
+        end
+
+        def get_collation_elements(collator, string)
+          iter = collator.get_collation_element_iterator(string)
+
+          collation_elements = []
+          ce = iter.next
+
+          while ce != Java::ComIbmIcuText::CollationElementIterator::NULLORDER
+            p1 = (ce >> 24) & LAST_BYTE_MASK
+            p2 = (ce >> 16) & LAST_BYTE_MASK
+
+            primary   = p2.zero? ? p1 : (p1 << 8) + p2
+            secondary = (ce >> 8) & LAST_BYTE_MASK
+            tertiarly = ce & LAST_BYTE_MASK
+
+            collation_elements << [primary, secondary, tertiarly]
+
+            ce = iter.next
+          end
+
+          collation_elements
+        end
+
+        def get_code_points(string)
+          TwitterCldr::Normalization::NFD.normalize_code_points(TwitterCldr::Utils::CodePoints.from_string(string))
+        end
+
+      end
+
+    end
+  end
+end