twitter_cldr 1.0.1 → 1.1.0

data/lib/normalizers/canonical/nfd.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+
+module TwitterCldr
+  module Normalizers
+    class NFD < Base
+      @@hangul_constants = {:SBase => "AC00".hex, :LBase => "1100".hex, :VBase => "1161".hex, :TBase => "11A7".hex,
+                            :Scount => 11172, :LCount => 19, :VCount => 21, :TCount => 28, :NCount => 588, :Scount => 1172}
+      class << self
+        def normalize(string)       
+          #Convert string to code points
+          code_points = string.split('').map { |char| char_to_code_point(char) }
+
+          #Normalize code points
+          normalized_code_points = normalize_code_points(code_points)
+
+          #Convert normalized code points back to string
+          normalized_code_points.map { |code_point| code_point_to_char(code_point) }.join
+        end
+
+        def normalize_code_points(code_points)          
+          code_points = code_points.map { |code_point| decompose code_point }.flatten
+          reorder code_points
+          code_points
+        end
+
+        #Recursively replace the given code point with the values in its Decomposition_Mapping property
+        def decompose(code_point)
+          unicode_data = TwitterCldr::Shared::UnicodeData.for_code_point(code_point)
+          return code_point unless unicode_data
+          decomposition_mapping = unicode_data.decomposition.split
+
+          # Special decomposition for Hangul syllables.
+          # Documented in Section 3.12 at http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
+          if unicode_data.name.include? 'Hangul'
+            sIndex = code_point.hex - @@hangul_constants[:SBase]
+
+            lIndex = sIndex / @@hangul_constants[:NCount]
+            vIndex = (sIndex % @@hangul_constants[:NCount]) / @@hangul_constants[:TCount]
+            tIndex = sIndex % @@hangul_constants[:TCount]
+
+            lPart = (@@hangul_constants[:LBase] + lIndex).to_s(16).upcase
+            vPart = (@@hangul_constants[:VBase] + vIndex).to_s(16).upcase
+            tPart = (@@hangul_constants[:TBase] + tIndex).to_s(16).upcase if tIndex > 0
+
+            [lPart, vPart, tPart].compact
+
+          #Return the code point if compatibility mapping or if no mapping exists
+          elsif decomposition_mapping.first =~ /<.*>/ || decomposition_mapping.empty?
+            code_point
+          else
+            decomposition_mapping.map do |decomposition_code_point|
+              decompose(decomposition_code_point)
+            end.flatten
+          end
+        end
+
+        #Swap any two adjacent code points A & B if ccc(A) > ccc(B) > 0
+        def reorder(code_points)
+          (code_points.size).times do
+            code_points.each_with_index do |cp, i|
+              unless i == (code_points.size - 1)
+                ccc_a, ccc_b = combining_class_for(cp), combining_class_for(code_points[i+1])
+                if (ccc_a > ccc_b) && (ccc_b > 0)
+                  code_points[i], code_points[i+1] = code_points[i+1], code_points[i]
+                end
+              end
+            end
+          end
+        end
+
+        def combining_class_for(code_point)
+          begin
+            unicode_data = TwitterCldr::Shared::UnicodeData.for_code_point(code_point).combining_class.to_i
+          rescue NoMethodError
+            0
+          end          
+        end
+      end
+    end
+  end
+end

data/lib/shared/currencies.rb

@@ -1,9 +1,12 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Shared
     class Currencies
-      @@resource = TwitterCldr.resources.resource_for("shared", "currencies")[:shared][:currencies]
+      @@resource = TwitterCldr.get_resource("shared", "currencies")[:shared][:currencies]
 
       class << self
         def countries

data/lib/shared/languages.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Shared
     class Languages
@@ -42,7 +45,7 @@ module TwitterCldr
 
         def get_resource(locale)
           locale = TwitterCldr.convert_locale(locale)
-          TwitterCldr.resources.resource_for(locale, "languages")[locale]
+          TwitterCldr.get_resource(locale, "languages")[locale]
         end
       end
     end

data/lib/shared/resources.rb

@@ -1,47 +1,27 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Shared
     class Resources
       def initialize
-        @resources_by_locale = {}
+        @resources_by_locale = Hash.new do |hash, locale|
+          hash[locale] = Hash.new { |h, resource| h[resource] = data_for(locale, resource) }
+        end
       end
 
       def resource_for(locale, resource)
-        locale = locale.to_sym
-        unless @resources_by_locale.include?(locale)
-          @resources_by_locale[locale] = {}
-        end
-
-        unless @resources_by_locale[locale].include?(resource)
-          @resources_by_locale[locale][resource] = data_for(locale, resource)
-        end
-
-        @resources_by_locale[locale][resource]
+        @resources_by_locale[locale.to_sym][resource]
       end
 
       protected
 
       def data_for(locale, resource)
-        deep_symbolize_keys(YAML.load(File.read(TwitterCldr.get_resource_file(locale, resource))))
+        TwitterCldr::Utils.deep_symbolize_keys(YAML.load(File.read(TwitterCldr.get_resource_file(locale, resource))))
       end
 
-      # adapted from: http://snippets.dzone.com/posts/show/11121 (first comment)
-      def deep_symbolize_keys(arg)
-        case arg
-          when Array then
-            arg.map { |elem| deep_symbolize_keys(elem) }
-          when Hash then
-            Hash[
-              arg.map do |key, value|
-                k = key.is_a?(String) ? key.to_sym : key
-                v = deep_symbolize_keys(value)
-                [k, v]
-              end]
-          else
-            arg
-        end
-      end
     end
   end
 end

data/lib/shared/timezones.rb

@@ -1,3 +1,6 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 # not yet implemented

data/lib/shared/unicode_data.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+    class UnicodeData
+      Attributes = Struct.new(:code_point, :name, :category, :combining_class, :bidi_class, :decomposition,
+                             :digit_value, :non_decimal_digit_value, :numeric_value, :bidi_mirrored, :unicode1_name,
+                             :iso_comment, :simple_uppercase_map, :simple_lowercase_map, :simple_titlecase_map)
+      class << self
+        def for_code_point(code_point)
+          blocks = TwitterCldr.get_resource("unicode_data", "blocks")
+          
+          #Find the target block
+          target = blocks.find do |block_name, range|
+            range.include? code_point.to_i(16)
+          end 
+
+          if target
+            block_data = TwitterCldr.get_resource("unicode_data", target.first)          
+            code_point_data = block_data.fetch(code_point.to_sym) { |code_point_sym| get_range_start(code_point_sym, block_data) }
+            Attributes.new(*code_point_data) if code_point_data
+          end
+        end
+
+        private
+        # Check if block constitutes a range. The code point beginning a range will have a name enclosed in <>, ending with 'First'
+        # eg: <CJK Ideograph Extension A, First>
+        # http://unicode.org/reports/tr44/#Code_Point_Ranges
+        def get_range_start(code_point, block_data)
+          start_code_point = block_data.keys.sort_by { |key| key.to_s.to_i(16) }.first
+          start_data = block_data[start_code_point].clone
+          if start_data[1] =~ /<.*, First>/
+            start_data[0] = code_point.to_s
+            start_data[1] = start_data[1].sub(', First', '')
+            start_data
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tokenizers/base.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class Base

data/lib/tokenizers/calendars/date_tokenizer.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class DateTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer

data/lib/tokenizers/calendars/datetime_tokenizer.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class DateTimeTokenizer < Base
@@ -32,7 +35,7 @@ module TwitterCldr
       protected
 
       def init_resources
-        @resource = TwitterCldr.resources.resource_for(@locale, "calendars")[TwitterCldr.convert_locale(@locale)]
+        @resource = TwitterCldr.get_resource(@locale, "calendars")[TwitterCldr.convert_locale(@locale)]
       end
 
       def init_placeholders

data/lib/tokenizers/calendars/time_tokenizer.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class TimeTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer

data/lib/tokenizers/key_path.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class KeyPath

data/lib/tokenizers/numbers/number_tokenizer.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class NumberTokenizer < Base
@@ -40,7 +43,7 @@ module TwitterCldr
       end
 
       def init_resources
-        @resource = TwitterCldr.resources.resource_for(@locale, "numbers")[TwitterCldr.convert_locale(@locale)]
+        @resource = TwitterCldr.get_resource(@locale, "numbers")[TwitterCldr.convert_locale(@locale)]
       end
 
       def pattern_for(resource)

data/lib/tokenizers/token.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
   module Tokenizers
     class Token

data/lib/twitter_cldr.rb

@@ -1,5 +1,8 @@
 # encoding: UTF-8
 
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 $:.push(File.dirname(__FILE__))
 
 $KCODE = 'UTF-8' unless RUBY_VERSION >= '1.9.0'
@@ -7,6 +10,7 @@ $KCODE = 'UTF-8' unless RUBY_VERSION >= '1.9.0'
 require 'yaml'
 require 'date'
 require 'time'
+require 'forwardable'
 
 require 'version'
 
@@ -20,12 +24,18 @@ require 'ext/numbers/bignum'
 require 'ext/numbers/fixnum'
 require 'ext/numbers/float'
 require 'ext/strings/symbol'
+require 'ext/strings/string'
+
+require 'utils'
 
 # manages access to CLDR resources (yaml files in resources dir)
 require 'shared/resources'
 
 
 module TwitterCldr
+
+  extend SingleForwardable
+
   DEFAULT_LOCALE = :en
   RESOURCE_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), "resources")
 
@@ -36,51 +46,59 @@ module TwitterCldr
 
   @@resources = TwitterCldr::Shared::Resources.new
 
-  def self.get_resource_file(locale, resource)
-    File.join(RESOURCE_DIR, self.convert_locale(locale).to_s, "#{resource}.yml")
-  end
+  def_delegator :resources, :resource_for, :get_resource
 
-  def self.resources
-    @@resources
-  end
+  class << self
 
-  def self.get_locale
-    if defined?(FastGettext)
-      locale = FastGettext.locale
-      locale = DEFAULT_LOCALE if locale.to_s.empty?
-    else
-      locale = DEFAULT_LOCALE
+    def get_resource_file(locale, resource)
+      File.join(RESOURCE_DIR, convert_locale(locale).to_s, "#{resource}.yml")
     end
 
-    (self.supported_locale?(locale) ? locale : DEFAULT_LOCALE).to_sym
-  end
+    def resources
+      @@resources
+    end
 
-  def self.convert_locale(locale)
-    locale = locale.to_sym
-    TWITTER_LOCALE_MAP.include?(locale) ? TWITTER_LOCALE_MAP[locale] : locale
-  end
+    def get_locale
+      if defined?(FastGettext)
+        locale = FastGettext.locale
+        locale = DEFAULT_LOCALE if locale.to_s.empty?
+      else
+        locale = DEFAULT_LOCALE
+      end
 
-  def self.supported_locales
-    unless defined?(@@supported_locales)
-      rejectable = [:shared]
-      @@supported_locales = Dir.glob(File.join(File.dirname(File.dirname(__FILE__)), "resources/*")).map do |file|
-        File.basename(file).to_sym
-      end.reject { |file| rejectable.include?(file) }
+      (supported_locale?(locale) ? locale : DEFAULT_LOCALE).to_sym
     end
 
-    @@supported_locales
-  end
+    def convert_locale(locale)
+      locale = locale.to_sym
+      TWITTER_LOCALE_MAP.include?(locale) ? TWITTER_LOCALE_MAP[locale] : locale
+    end
+
+    def supported_locales
+      unless defined?(@@supported_locales)
+        rejectable = [:shared]
+        @@supported_locales = Dir.glob(File.join(File.dirname(File.dirname(__FILE__)), "resources/*")).map do |file|
+          File.basename(file).to_sym
+        end.reject { |file| rejectable.include?(file) }
+      end
+
+      @@supported_locales
+    end
+
+    def supported_locale?(locale)
+      locale = locale.to_sym
+      supported_locales.include?(locale) || supported_locales.include?(convert_locale(locale))
+    end
 
-  def self.supported_locale?(locale)
-    locale = locale.to_sym
-    self.supported_locales.include?(locale) || self.supported_locales.include?(self.convert_locale(locale))
   end
+
 end
 
 
 # other shared libraries (most access shared resource data in resources/shared)
 require 'shared/currencies'
 require 'shared/languages'
+require 'shared/unicode_data'
 
 # all tokenizers
 require 'tokenizers/base'
@@ -100,9 +118,14 @@ require 'formatters/numbers/number_formatter'
 require 'formatters/numbers/decimal_formatter'
 require 'formatters/numbers/currency_formatter'
 require 'formatters/numbers/percent_formatter'
+require 'formatters/plurals/plural_formatter'
 require 'formatters/plurals/rules'
 
 # formatter helpers
 require 'formatters/numbers/helpers/base'
 require 'formatters/numbers/helpers/fraction'
 require 'formatters/numbers/helpers/integer'
+
+# all normalizers
+require 'normalizers/base'
+require 'normalizers/canonical/nfd'

data/lib/utils/interpolation.rb

@@ -0,0 +1,105 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# The implementation of the TwitterCldr.interpolate method that backports String interpolation capabilities
+# (originally implemented in String#% method) from Ruby 1.9 to Ruby 1.8 is heavily influenced by the
+# implementation of the same feature in i18n (https://github.com/svenfuchs/i18n/blob/89ea337f48562370988421e50caa7c2fe89452c7/lib/i18n/core_ext/string/interpolate.rb)
+# and gettext (https://github.com/mutoh/gettext/blob/11b8c1525ba9f00afb1942f7ebf34bec12f7558b/lib/gettext/core_ext/string.rb) gems.
+#
+# See NOTICE file for corresponding license agreements.
+
+
+# KeyError is raised during interpolation when there is a placeholder that doesn't have corresponding key in the
+# interpolation hash. KeyError is defined in 1.9. We define it for prior versions of Ruby to have the same behavior.
+#
+class KeyError < IndexError
+  def initialize(message = nil)
+    super(message || 'key not found')
+  end
+end unless defined?(KeyError)
+
+
+module TwitterCldr
+  module Utils
+
+    HASH_INTERPOLATION_REGEXP = Regexp.union(
+        /%\{(\w+)\}/,
+        /%<(\w+)>(.*?\d*\.?\d*[bBdiouxXeEfgGcps])/
+    )
+
+    HASH_INTERPOLATION_WITH_ESCAPE_REGEXP = Regexp.union(
+        /%%/,
+        HASH_INTERPOLATION_REGEXP
+    )
+
+    class << self
+
+      # Uses +string+ as a format specification and returns the result of applying it to +args+.
+      #
+      # There are three ways to use it:
+      #
+      # * Using a single argument or Array of arguments.
+      #
+      #   This is the default behaviour of the String#% method. See Kernel#sprintf for more details about the format
+      #   specification.
+      #
+      #   Example:
+      #
+      #     TwitterCldr::Utils.interpolate('%d %s', [1, 'message'])
+      #     # => "1 message"
+      #
+      # * Using a Hash as an argument and unformatted, named placeholders (Ruby 1.9 syntax).
+      #
+      #   When you pass a Hash as an argument and specify placeholders with %{foo} it will interpret the hash values as
+      #   named arguments.
+      #
+      #   Example:
+      #
+      #     TwitterCldr::Utils.interpolate('%{firstname}, %{lastname}', :firstname => 'Masao', :lastname => 'Mutoh')
+      #     # => "Masao Mutoh"
+      #
+      # * Using a Hash as an argument and formatted, named placeholders (Ruby 1.9 syntax).
+      #
+      #   When you pass a Hash as an argument and specify placeholders with %<foo>d  it will interpret the hash values
+      #   as named arguments and format the value according to the formatting instruction appended to the closing >.
+      #
+      #   Example:
+      #
+      #     TwitterCldr::Utils.interpolate('%<integer>d, %<float>.1f', :integer => 10, :float => 43.4)
+      #     # => "10, 43.3"
+      #
+      # An exception can be thrown in two cases when Ruby 1.9 interpolation syntax is used:
+      #
+      # * ArgumentError is thrown if Ruby 1.9. interpolation syntax is used in +string+, but +args+ is not a Hash;
+      # * KeyError is thrown if the value for one of the placeholders in +string+ is missing in +args+ hash.
+      #
+      def interpolate(string, args)
+        string =~ HASH_INTERPOLATION_REGEXP ? interpolate_hash(string, args) : interpolate_value_or_array(string, args)
+      end
+
+      private
+
+      def interpolate_hash(string, args)
+        raise ArgumentError.new('expected a Hash') unless args.is_a?(Hash)
+
+        string.gsub(HASH_INTERPOLATION_WITH_ESCAPE_REGEXP) do |match|
+          if match == '%%'
+            '%'
+          else
+            key = ($1 || $2).to_sym
+            raise KeyError unless args.has_key?(key)
+            $3 ? sprintf("%#{$3}", args[key]) : args[key]
+          end
+        end
+      end
+
+      def interpolate_value_or_array(string, args)
+        string.gsub(/%([{<])/, '%%\1') % args
+      end
+
+    end
+
+  end
+end

data/lib/utils.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'utils/interpolation'
+
+module TwitterCldr
+  module Utils
+
+    class << self
+
+      # adapted from: http://snippets.dzone.com/posts/show/11121 (first comment)
+      def deep_symbolize_keys(arg)
+        case arg
+          when Array
+            arg.map { |elem| deep_symbolize_keys(elem) }
+          when Hash
+            Hash[arg.map { |k, v| [k.is_a?(String) ? k.to_sym : k, deep_symbolize_keys(v)] }]
+          else
+            arg
+        end
+      end
+
+    end
+
+  end
+end

data/lib/version.rb

@@ -1,3 +1,8 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
 module TwitterCldr
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end