twitter_cldr 1.4.1 → 1.5.0

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Collation
+
+    # Builds a collation elements Trie from the file containing a fractional collation elements table.
+    #
+    class TrieBuilder
+
+      FRACTIONAL_UCA_REGEXP = /^((?:[0-9A-F]+)(?:\s[0-9A-F]+)*);\s((?:\[.*?\])(?:\[.*?\])*)/
+
+      def self.load_trie(file_path)
+        new(file_path).build
+      end
+
+      def initialize(resource)
+        @file_path = File.join(TwitterCldr::RESOURCES_DIR, resource)
+      end
+
+      def build
+        parse_trie(load_collation_elements_table)
+      end
+
+      private
+
+      def parse_trie(table)
+        trie = TwitterCldr::Collation::Trie.new
+
+        table.lines.each do |line|
+          trie.add(parse_code_points($1), parse_collation_element($2)) if FRACTIONAL_UCA_REGEXP =~ line
+        end
+
+        trie
+      end
+
+      def load_collation_elements_table
+        open(@file_path, 'r')
+      end
+
+      def parse_code_points(string)
+        string.split.map { |cp| cp.to_i(16) }
+      end
+
+      def parse_collation_element(string)
+        string.scan(/\[.*?\]/).map do |match|
+          match[1..-2].gsub(/\s/, '').split(',', -1).map { |bytes| bytes.to_i(16) }
+        end
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/collation.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Collation
+    autoload :Collator,                  'twitter_cldr/collation/collator'
+    autoload :ImplicitCollationElements, 'twitter_cldr/collation/implicit_collation_elements'
+    autoload :SortKey,                   'twitter_cldr/collation/sort_key'
+    autoload :Trie,                      'twitter_cldr/collation/trie'
+    autoload :TrieBuilder,               'twitter_cldr/collation/trie_builder'
+  end
+end

data/lib/twitter_cldr/core_ext/localized_object.rb

@@ -9,10 +9,11 @@ module TwitterCldr
 
     def initialize(obj, locale, options = {})
       @base_obj = obj
-      @locale = locale
+      @locale = TwitterCldr.convert_locale(locale)
+      @locale = TwitterCldr::DEFAULT_LOCALE unless TwitterCldr.supported_locale?(@locale)
 
       options = options.dup
-      options[:locale] ||= @locale
+      options[:locale] = @locale
 
       @formatter = formatter_const.new(options) if formatter_const
     end

data/lib/twitter_cldr/core_ext/string.rb

@@ -32,7 +32,7 @@ module TwitterCldr
       options[:using] ||= :NFD
 
       if VALID_NORMALIZERS.include?(options[:using])
-        normalizer_const = TwitterCldr::Normalizers.const_get(options[:using])
+        normalizer_const = TwitterCldr::Normalization.const_get(options[:using])
         LocalizedString.new(normalizer_const.normalize(@base_obj), @locale)
       else
         raise ArgumentError.new("Invalid normalization form specified with :using option.  Choices are [#{VALID_NORMALIZERS.map(&:to_s).join(", ")}]")

data/lib/twitter_cldr/formatters/calendars/datetime_formatter.rb

@@ -51,7 +51,13 @@ module TwitterCldr
       protected
 
       def era(date, pattern, length)
-        raise NotImplementedError
+        choices = case length
+          when 1..3
+            @tokenizer.calendar[:eras][:abbr]
+          else
+            @tokenizer.calendar[:eras][:name]
+        end
+        choices[date.year < 0 ? 0 : 1]
       end
 
       def year(date, pattern, length)
@@ -72,100 +78,111 @@ module TwitterCldr
       def quarter(date, pattern, length)
         quarter = (date.month.to_i - 1) / 3 + 1
         case length
-        when 1
-          quarter.to_s
-        when 2
-          quarter.to_s.rjust(length, '0')
-        when 3
-          @tokenizer.calendar[:quarters][:format][:abbreviated][quarter]
-        when 4
-          @tokenizer.calendar[:quarters][:format][:wide][quarter]
+          when 1
+            quarter.to_s
+          when 2
+            quarter.to_s.rjust(length, '0')
+          when 3
+            @tokenizer.calendar[:quarters][:format][:abbreviated][quarter]
+          when 4
+            @tokenizer.calendar[:quarters][:format][:wide][quarter]
         end
       end
 
       def quarter_stand_alone(date, pattern, length)
         quarter = (date.month.to_i - 1) / 3 + 1
         case length
-        when 1
-          quarter.to_s
-        when 2
-          quarter.to_s.rjust(length, '0')
-        when 3
-          raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
-          # @tokenizer.calendar[:quarters][:'stand-alone'][:abbreviated][key]
-        when 4
-          raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
-          # @tokenizer.calendar[:quarters][:'stand-alone'][:wide][key]
-        when 5
-          @tokenizer.calendar[:quarters][:'stand-alone'][:narrow][quarter]
+          when 1
+            quarter.to_s
+          when 2
+            quarter.to_s.rjust(length, '0')
+          when 3
+            raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
+            # @tokenizer.calendar[:quarters][:'stand-alone'][:abbreviated][key]
+          when 4
+            raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
+            # @tokenizer.calendar[:quarters][:'stand-alone'][:wide][key]
+          when 5
+            @tokenizer.calendar[:quarters][:'stand-alone'][:narrow][quarter]
         end
       end
 
       def month(date, pattern, length)
         case length
-        when 1
-          date.month.to_s
-        when 2
-          date.month.to_s.rjust(length, '0')
-        when 3
-          @tokenizer.calendar[:months][:format][:abbreviated][date.month]
-        when 4
-          @tokenizer.calendar[:months][:format][:wide][date.month]
-        when 5
-          raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
-          # @tokenizer.calendar[:months][:format][:narrow][date.month]
-        else
-          # raise unknown date format
+          when 1
+            date.month.to_s
+          when 2
+            date.month.to_s.rjust(length, '0')
+          when 3
+            @tokenizer.calendar[:months][:format][:abbreviated][date.month]
+          when 4
+            @tokenizer.calendar[:months][:format][:wide][date.month]
+          when 5
+            raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
+            # @tokenizer.calendar[:months][:format][:narrow][date.month]
+          else
+            # raise unknown date format
         end
       end
 
       def month_stand_alone(date, pattern, length)
         case length
-        when 1
-          date.month.to_s
-        when 2
-          date.month.to_s.rjust(length, '0')
-        when 3
-          raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
-          @tokenizer.calendar[:months][:'stand-alone'][:abbreviated][date.month]
-        when 4
-          raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
-          @tokenizer.calendar[:months][:'stand-alone'][:wide][date.month]
-        when 5
-          @tokenizer.calendar[:months][:'stand-alone'][:narrow][date.month]
-        else
-          # raise unknown date format
+          when 1
+            date.month.to_s
+          when 2
+            date.month.to_s.rjust(length, '0')
+          when 3
+            raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
+            @tokenizer.calendar[:months][:'stand-alone'][:abbreviated][date.month]
+          when 4
+            raise NotImplementedError, 'requires cldr\'s "multiple inheritance"'
+            @tokenizer.calendar[:months][:'stand-alone'][:wide][date.month]
+          when 5
+            @tokenizer.calendar[:months][:'stand-alone'][:narrow][date.month]
+          else
+            # raise unknown date format
         end
       end
 
       def day(date, pattern, length)
         case length
-        when 1
-          date.day.to_s
-        when 2
-          date.day.to_s.rjust(length, '0')
+          when 1
+            date.day.to_s
+          when 2
+            date.day.to_s.rjust(length, '0')
         end
       end
 
       def weekday(date, pattern, length)
         key = WEEKDAY_KEYS[date.wday]
         case length
-        when 1..3
-          @tokenizer.calendar[:days][:format][:abbreviated][key]
-        when 4
-          @tokenizer.calendar[:days][:format][:wide][key]
-        when 5
-          @tokenizer.calendar[:days][:'stand-alone'][:narrow][key]
+          when 1..3
+            @tokenizer.calendar[:days][:format][:abbreviated][key]
+          when 4
+            @tokenizer.calendar[:days][:format][:wide][key]
+          when 5
+            @tokenizer.calendar[:days][:'stand-alone'][:narrow][key]
         end
       end
 
       def weekday_local(date, pattern, length)
         # "Like E except adds a numeric value depending on the local starting day of the week"
-        raise NotImplementedError, 'need to defer a country to lookup the local first day of week from weekdata'
+        # CLDR does not contain data as to which day is the first day of the week, so we will assume Monday (Ruby default)
+        case length
+          when 1..2
+            date.cwday.to_s
+          else
+            weekday(date, pattern, length)
+        end
       end
 
       def weekday_local_stand_alone(date, pattern, length)
-        raise NotImplementedError, 'need to defer a country to lookup the local first day of week from weekdata'
+        case length
+          when 1
+            weekday_local(date, pattern, length)
+          else
+            weekday(date, pattern, length)
+        end
       end
 
       def period(time, pattern, length)
@@ -178,14 +195,14 @@ module TwitterCldr
       def hour(time, pattern, length)
         hour = time.hour
         hour = case pattern[0, 1]
-        when 'h' # [1-12]
-          hour > 12 ? (hour - 12) : (hour == 0 ? 12 : hour)
-        when 'H' # [0-23]
-          hour
-        when 'K' # [0-11]
-          hour > 11 ? hour - 12 : hour
-        when 'k' # [1-24]
-          hour == 0 ? 24 : hour
+          when 'h' # [1-12]
+            hour > 12 ? (hour - 12) : (hour == 0 ? 12 : hour)
+          when 'H' # [0-23]
+            hour
+          when 'K' # [0-11]
+            hour > 11 ? hour - 12 : hour
+          when 'k' # [1-24]
+            hour == 0 ? 24 : hour
         end
         length == 1 ? hour.to_s : hour.to_s.rjust(length, '0')
       end
@@ -205,10 +222,10 @@ module TwitterCldr
 
       def timezone(time, pattern, length)
         case length
-        when 1..3
-          time.zone
-        else
-          "UTC #{time.strftime("%z")}"
+          when 1..3
+            time.zone
+          else
+            "UTC #{time.strftime("%z")}"
         end
       end
 

data/lib/twitter_cldr/normalization/base.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Normalization
+    class Base
+
+      class << self
+
+        def combining_class_for(code_point)
+          TwitterCldr::Shared::CodePoint.for_hex(code_point).combining_class.to_i
+        rescue NoMethodError
+          0
+        end
+
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/normalization/hangul.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Normalization
+    module Hangul
+
+      class << self
+
+        # Special composition for Hangul syllables. Documented in Section 3.12 at
+        # http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
+        #
+        def compose(code_points)
+          l = code_points.first - LBASE
+          v = code_points[1] - VBASE
+          t = code_points[2] ? code_points[2] - TBASE : 0  # T part may be missing, that's ok
+
+          SBASE + l * NCOUNT + v * TCOUNT + t
+        end
+
+        # Special decomposition for Hangul syllables. Documented in Section 3.12 at http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
+        # Also see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm#Hangul_Implicit_CEs
+        #
+        def decompose(code_point)
+          l = code_point - SBASE
+
+          t = l % TCOUNT
+          l /= TCOUNT
+          v = l % VCOUNT
+          l /= VCOUNT
+
+          result = []
+
+          result << LBASE + l
+          result << VBASE + v
+          result << TBASE + t if t > 0
+
+          result
+        end
+
+        def hangul_syllable?(code_point)
+          (SBASE...SLIMIT).include?(code_point)
+        end
+
+        SBASE  = 0xAC00
+        LBASE  = 0x1100
+        VBASE  = 0x1161
+        TBASE  = 0x11A7
+
+        LCOUNT = 19
+        VCOUNT = 21
+        TCOUNT = 28
+
+        NCOUNT = VCOUNT * TCOUNT # 588
+        SCOUNT = LCOUNT * NCOUNT # 11172
+
+        LLIMIT = LBASE + LCOUNT  # 0x1113 = 4371
+        VLIMIT = VBASE + VCOUNT  # 0x1176 = 4470
+        TLIMIT = TBASE + TCOUNT  # 0x11C3 = 4547
+        SLIMIT = SBASE + SCOUNT  # 0xD7A4 = 55204
+
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/{normalizers → normalization}/nfc.rb

@@ -4,7 +4,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  module Normalizers
+  module Normalization
 
     # Implements normalization of a Unicode string to Normalization Form C (NFC).
     # This normalization includes canonical decomposition followed by canonical composition.
@@ -14,7 +14,7 @@ module TwitterCldr
       class << self
 
         def normalize_code_points(code_points)
-          compose(TwitterCldr::Normalizers::NFD.normalize_code_points(code_points))
+          compose(TwitterCldr::Normalization::NFD.normalize_code_points(code_points))
         end
 
       end

data/lib/twitter_cldr/{normalizers → normalization}/nfd.rb

@@ -4,7 +4,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  module Normalizers
+  module Normalization
 
     # Implements normalization of a Unicode string to Normalization Form D (NFD).
     # This normalization includes only canonical decomposition.

data/lib/twitter_cldr/{normalizers → normalization}/nfkc.rb

@@ -4,7 +4,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  module Normalizers
+  module Normalization
 
     # Implements normalization of a Unicode string to Normalization Form KC (NFKC).
     # This normalization form includes compatibility decomposition followed by compatibility composition.
@@ -20,7 +20,7 @@ module TwitterCldr
         end
 
         def normalize_code_points(code_points)
-          compose(TwitterCldr::Normalizers::NFKD.normalize_code_points(code_points))
+          compose(TwitterCldr::Normalization::NFKD.normalize_code_points(code_points))
         end
 
         protected
@@ -44,7 +44,7 @@ module TwitterCldr
             end
 
             if hangul_code_points.size > 1 && !next_hangul_type
-              hangul_code_points.size.times { final.pop }
+              final.pop(hangul_code_points.size)
               final << compose_hangul(hangul_code_points)
               hangul_code_points.clear
             end
@@ -55,23 +55,11 @@ module TwitterCldr
         end
 
         def valid_hangul_sequence?(buffer_size, hangul_type)
-          case [buffer_size, hangul_type]
-            when [0, :lparts], [1, :vparts], [2, :tparts]
-              true
-            else
-              false
-          end
+          [[0, :lparts], [1, :vparts], [2, :tparts]].include?([buffer_size, hangul_type])
         end
 
-        # Special composition for Hangul syllables. Documented in Section 3.12 at
-        # http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
-        #
         def compose_hangul(code_points)
-          l_index = code_points.first.hex - HANGUL_DECOMPOSITION_CONSTANTS[:LBase]
-          v_index = code_points[1].hex - HANGUL_DECOMPOSITION_CONSTANTS[:VBase]
-          t_index = code_points[2] ? code_points[2].hex - HANGUL_DECOMPOSITION_CONSTANTS[:TBase] : 0  # tpart may be missing, that's ok
-          lv_index = (l_index * HANGUL_DECOMPOSITION_CONSTANTS[:NCount]) + (v_index * HANGUL_DECOMPOSITION_CONSTANTS[:TCount])
-          (HANGUL_DECOMPOSITION_CONSTANTS[:SBase] + lv_index + t_index).to_s(16).upcase.rjust(4, "0")
+          TwitterCldr::Normalization::Hangul.compose(code_points.map { |cp| cp.hex }).to_s(16).upcase.rjust(4, "0")
         end
 
         # Implements composition of Unicode code points following the guidelines here:

data/lib/twitter_cldr/{normalizers → normalization}/nfkd.rb

@@ -4,12 +4,12 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  # Normalizers module includes algorithm for Unicode normalization. Basic information on this topic can be found in the
+  # Normalization module includes algorithm for Unicode normalization. Basic information on this topic can be found in the
   # Unicode Standard Annex #15 "Unicode Normalization Forms" at http://www.unicode.org/reports/tr15/. More detailed
   # description is given in the section "3.11 Normalization Forms" of the Unicode Standard core specification. The
   # latest version at the moment (for Unicode 6.1) is available at http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf.
   #
-  module Normalizers
+  module Normalization
 
     # Implements normalization of a Unicode string to Normalization Form KD (NFKD).
     # This normalization form includes only compatibility decomposition.
@@ -73,23 +73,8 @@ module TwitterCldr
           unicode_data.decomposition.split
         end
 
-        # Special decomposition for Hangul syllables. Documented in Section 3.12 at
-        # http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
-        #
         def decompose_hangul(code_point)
-          s_index = code_point.hex - HANGUL_DECOMPOSITION_CONSTANTS[:SBase]
-
-          l_index = s_index / HANGUL_DECOMPOSITION_CONSTANTS[:NCount]
-          v_index = (s_index % HANGUL_DECOMPOSITION_CONSTANTS[:NCount]) / HANGUL_DECOMPOSITION_CONSTANTS[:TCount]
-          t_index = s_index % HANGUL_DECOMPOSITION_CONSTANTS[:TCount]
-
-          result = []
-
-          result << (HANGUL_DECOMPOSITION_CONSTANTS[:LBase] + l_index).to_s(16).upcase
-          result << (HANGUL_DECOMPOSITION_CONSTANTS[:VBase] + v_index).to_s(16).upcase
-          result << (HANGUL_DECOMPOSITION_CONSTANTS[:TBase] + t_index).to_s(16).upcase if t_index > 0
-
-          result
+          TwitterCldr::Normalization::Hangul.decompose(code_point.hex).map { |e| e.to_s(16).upcase }
         end
 
         # Performs the Canonical Ordering Algorithm by stable sorting of every subsequence of combining code points

data/lib/twitter_cldr/normalization.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Normalization
+    autoload :Base,   'twitter_cldr/normalization/base'
+    autoload :Hangul, 'twitter_cldr/normalization/hangul'
+    autoload :NFC,    'twitter_cldr/normalization/nfc'
+    autoload :NFD,    'twitter_cldr/normalization/nfd'
+    autoload :NFKC,   'twitter_cldr/normalization/nfkc'
+    autoload :NFKD,   'twitter_cldr/normalization/nfkd'
+  end
+end

data/lib/twitter_cldr/shared/code_point.rb

@@ -38,7 +38,9 @@ module TwitterCldr
       class << self
 
         def for_hex(code_point)
-          target = get_block(code_point.rjust(4, "0").upcase)
+          code_point = code_point.rjust(4, '0').upcase
+
+          target = get_block(code_point)
 
           if target && target.first
             block_data = TwitterCldr.get_resource(:unicode_data, target.first)
@@ -63,7 +65,7 @@ module TwitterCldr
         def hangul_type(code_point)
           if code_point
             code_point_int = code_point.hex
-            [:lparts, :vparts, :tparts, :compositions, :decompositions].each do |type|
+            [:lparts, :vparts, :tparts, :compositions].each do |type|
               hangul_blocks[type].each do |range|
                 return type if range.include?(code_point_int)
               end
@@ -80,7 +82,7 @@ module TwitterCldr
         protected
 
         def hangul_blocks
-          @hangul_blocks ||= TwitterCldr.get_resource(:unicode_data, :blocks_hangul)
+          @hangul_blocks ||= TwitterCldr.get_resource(:unicode_data, :hangul_blocks)
         end
 
         def composition_exclusions

data/lib/twitter_cldr/tokenizers/base.rb

@@ -11,7 +11,7 @@ module TwitterCldr
       attr_accessor :type, :placeholders
 
       def initialize(options = {})
-        @locale = (options[:locale] || TwitterCldr::DEFAULT_LOCALE).to_sym
+        @locale = TwitterCldr.convert_locale(options[:locale] || TwitterCldr::DEFAULT_LOCALE)
         self.init_resources
         self.init_placeholders
       end
@@ -100,6 +100,20 @@ module TwitterCldr
         end
       end
 
+      # expands all path symbols
+      def expand(current, haystack)
+        if current.is_a?(Symbol)
+          expand(traverse(current.to_s.split('.').map(&:to_sym), haystack), haystack)
+        elsif current.is_a?(Hash)
+          current.inject({}) do |ret, (key, val)|
+            ret[key] = expand(val, haystack)
+            ret
+          end
+        else
+          current
+        end
+      end
+
       def expand_pattern(format_str, type)
         if format_str.is_a?(Symbol)
           # symbols mean another path was given

data/lib/twitter_cldr/tokenizers/calendars/datetime_tokenizer.rb

@@ -45,7 +45,8 @@ module TwitterCldr
       end
 
       def init_resources
-        @resource = TwitterCldr.get_locale_resource(@locale, :calendars)[TwitterCldr.convert_locale(@locale)]
+        @resource = TwitterCldr.get_locale_resource(@locale, :calendars)[@locale]
+        @resource = expand(@resource, @resource)
 
         @resource[:calendars].each_pair do |calendar_type, options|
           next if calendar_type == DEFAULT_CALENDAR_TYPE
@@ -78,6 +79,10 @@ module TwitterCldr
       def pattern_for(resource)
         resource.is_a?(Hash) ? resource[:pattern] : resource
       end
+
+      def path_map
+        PATH_MAP
+      end
     end
   end
 end

data/lib/twitter_cldr/utils/code_points.rb

@@ -10,7 +10,7 @@ module TwitterCldr
       class << self
 
         def to_char(code_point)
-          [code_point.upcase.hex].pack('U*')
+          [code_point.hex].pack('U*')
         end
 
         def from_char(char)

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "1.4.1"
-end
+  VERSION = "1.5.0"
+end