twitter_cldr 1.9.1 → 2.0.0

data/lib/twitter_cldr/shared/currencies.rb

@@ -6,34 +6,41 @@
 module TwitterCldr
   module Shared
     module Currencies
-
       class << self
-
         def countries
-          resource.keys.map(&:to_s)
+          Kernel.warn("Currencies#countries will be deprecated. Please use Currencies#for_code.")
+          resource_countries.keys.map(&:to_s)
         end
 
-        def currency_codes
-          resource.values.map { |data| data[:code] }
+        def currency_codes(locale = :en)
+          resource(locale).keys.map { |c| c.to_s }
         end
 
-        def for_country(country_name)
-          resource[country_name.to_sym]
+        def for_country(country_name, locale = :en)
+          Kernel.warn("Currencies#for_country will be deprecated. Please use Currencies#for_code.")
+          for_code(resource_countries[country_name.to_sym][:code], locale) if resource_countries[country_name.to_sym]
         end
 
-        def for_code(currency_code)
-          country_name, data = resource.detect { |_, data| data[:code] == currency_code }
-          { :country => country_name.to_s, :currency => data[:currency], :symbol => data[:symbol] } if data
+        def for_code(currency_code, locale = :en)
+          currency_code = currency_code.to_sym
+          data = resource(locale)[currency_code]
+          { :currency => currency_code,
+            :name => data[:one],
+            :symbol => data[:symbol] } if data
         end
 
         private
 
-        def resource
-          @resource ||= TwitterCldr.get_resource(:shared, :currencies)
+        def resource_countries
+          @resource_countries ||= TwitterCldr.get_resource(:shared, :currencies)
         end
 
+        def resource(locale)
+          locale = locale.to_sym
+          @resource ||= {}
+          @resource[locale] ||= TwitterCldr.get_resource(:locales, locale, :currencies)[locale][:currencies]
+        end
       end
-
     end
   end
 end

data/lib/twitter_cldr/tokenizers.rb

@@ -5,14 +5,15 @@
 
 module TwitterCldr
   module Tokenizers
-    autoload :Base,              'twitter_cldr/tokenizers/base'
-    autoload :CompositeToken,    'twitter_cldr/tokenizers/composite_token'
-    autoload :KeyPath,           'twitter_cldr/tokenizers/key_path'
-    autoload :Token,             'twitter_cldr/tokenizers/token'
-    autoload :DateTimeTokenizer, 'twitter_cldr/tokenizers/calendars/datetime_tokenizer'
-    autoload :DateTokenizer,     'twitter_cldr/tokenizers/calendars/date_tokenizer'
-    autoload :TimeTokenizer,     'twitter_cldr/tokenizers/calendars/time_tokenizer'
-    autoload :NumberTokenizer,   'twitter_cldr/tokenizers/numbers/number_tokenizer'
-    autoload :TimespanTokenizer,  'twitter_cldr/tokenizers/calendars/timespan_tokenizer'
+    autoload :Base,                         'twitter_cldr/tokenizers/base'
+    autoload :CompositeToken,               'twitter_cldr/tokenizers/composite_token'
+    autoload :KeyPath,                      'twitter_cldr/tokenizers/key_path'
+    autoload :Token,                        'twitter_cldr/tokenizers/token'
+    autoload :DateTimeTokenizer,            'twitter_cldr/tokenizers/calendars/datetime_tokenizer'
+    autoload :DateTokenizer,                'twitter_cldr/tokenizers/calendars/date_tokenizer'
+    autoload :TimeTokenizer,                'twitter_cldr/tokenizers/calendars/time_tokenizer'
+    autoload :NumberTokenizer,              'twitter_cldr/tokenizers/numbers/number_tokenizer'
+    autoload :TimespanTokenizer,            'twitter_cldr/tokenizers/calendars/timespan_tokenizer'
+    autoload :AdditionalDateFormatSelector, 'twitter_cldr/tokenizers/calendars/additional_date_format_selector'
   end
 end

data/lib/twitter_cldr/tokenizers/base.rb

@@ -7,8 +7,8 @@ module TwitterCldr
   module Tokenizers
     class Base
       attr_reader :resource, :locale
-      attr_reader :token_splitter_regex, :token_type_regexes, :paths
-      attr_accessor :type, :placeholders
+      attr_reader :token_splitter_regexes, :token_type_regexes, :paths
+      attr_accessor :type, :format, :placeholders
 
       def initialize(options = {})
         @locale = TwitterCldr.convert_locale(options[:locale] || TwitterCldr::DEFAULT_LOCALE)
@@ -21,33 +21,45 @@ module TwitterCldr
       # Not to be confused with tokenize_pattern, which pulls out placeholders.  Tokenize_format actually splits a completely
       # expanded format string into whatever parts are defined by the subclass's token type and token splitter regexes.
       def tokenize_format(text)
-        final = []
-        text.split(token_splitter_regex).each_with_index do |token, index|
+        text.split(token_splitter_regex_for(type)).each_with_index.inject([]) do |ret, (token, index)|
           unless index == 0 && token == ""
-            token_type_regexes.each do |token_type|
-              if token =~ token_type[:regex]
-                if token_type[:type] == :composite
-                  content = token.match(token_type[:content])[1]
-                  final << CompositeToken.new(tokenize_format(content))
-                else
-                  final << Token.new(:value => token, :type => token_type[:type])
-                end
-
-                break
-              end
+            regexes = token_type_regexes_for(type)
+
+            token_type = regexes.inject([]) do |match_ret, (token_type, matchers)|
+              match_ret << token_type if token =~ matchers[:regex]
+              match_ret
+            end.min { |a, b| regexes[a][:priority] <=> regexes[b][:priority] }
+
+            if token_type == :composite
+              content = token.match(regexes[token_type][:content])[1]
+              ret << CompositeToken.new(tokenize_format(content))
+            else
+              ret << Token.new(:value => token, :type => token_type)
             end
           end
+          ret
         end
-        final
       end
 
-      def tokens_for(path, type)
+      def token_type_regexes_for(type)
+        token_type_regexes[type] || token_type_regexes[:else]
+      end
+
+      def token_splitter_regex_for(type)
+        token_splitter_regexes[type] || token_splitter_regexes[:else]
+      end
+
+      def tokens_for(path, additional_cache_key_params = [])
+        tokens_for_pattern(pattern_for(traverse(path)), path, additional_cache_key_params)
+      end
+
+      def tokens_for_pattern(pattern, path, additional_cache_key_params = [])
         @@token_cache ||= {}
-        cache_key = TwitterCldr::Utils.compute_cache_key(@locale, path.join('.'), type)
+        cache_key = TwitterCldr::Utils.compute_cache_key(@locale, path.join('.'), type, format || "nil", *additional_cache_key_params)
 
         unless @@token_cache.include?(cache_key)
           result = []
-          tokens = expand_pattern(pattern_for(traverse(path)), type)
+          tokens = expand_pattern(pattern)
 
           tokens.each do |token|
             if token.is_a?(Token) || token.is_a?(CompositeToken)
@@ -108,10 +120,10 @@ module TwitterCldr
         end
       end
 
-      def expand_pattern(format_str, type)
+      def expand_pattern(format_str)
         if format_str.is_a?(Symbol)
           # symbols mean another path was given
-          expand_pattern(pattern_for(traverse(format_str.to_s.split('.').map(&:to_sym))), type)
+          expand_pattern(pattern_for(traverse(format_str.to_s.split('.').map(&:to_sym))))
         else
           parts = tokenize_pattern(format_str)
           final = []

data/lib/twitter_cldr/tokenizers/calendars/additional_date_format_selector.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class AdditionalDateFormatSelector
+      attr_reader :pattern_hash
+
+      def initialize(pattern_hash)
+        @pattern_hash = pattern_hash
+      end
+
+      def find_closest(goal_pattern)
+        if !goal_pattern || goal_pattern.strip.empty?
+          nil
+        else
+          rank(goal_pattern).min do |(p1, score1), (p2, score2)|
+            score1 <=> score2
+          end.first
+        end
+      end
+
+      def patterns
+        @pattern_hash.keys
+      end
+
+      protected
+
+      def separate(pattern_key)
+        last_char = ""
+        pattern_key.each_char.each_with_index.inject([]) do |ret, (char, index)|
+          char == last_char ? ret[-1] += char : ret << char
+          last_char = char
+          ret
+        end
+      end
+
+      def all_separated_patterns
+        @separated_patterns ||= @pattern_hash.map { |pattern, _| separate(pattern.to_s) }
+      end
+
+      def score(entities, goal_entities)
+        # weight existence a little more heavily than the others
+        score = exist_score(entities, goal_entities) * 2
+        score += position_score(entities, goal_entities)
+        score + count_score(entities, goal_entities)
+      end
+
+      def position_score(entities, goal_entities)
+        goal_entities.each_with_index.inject(0) do |sum, (goal_entity, index)|
+          if found = entities.index(goal_entity)
+            sum + (found - index).abs
+          else
+            sum
+          end
+        end
+      end
+
+      def exist_score(entities, goal_entities)
+        goal_entities.count do |goal_entity|
+          !entities.any? { |entity| entity[0] == goal_entity[0] }
+        end
+      end
+
+      def count_score(entities, goal_entities)
+        goal_entities.inject(0) do |sum, goal_entity|
+          if found_entity = entities.select { |entity| entity[0] == goal_entity[0] }.first
+            sum + (found_entity.size - goal_entity.size).abs
+          else
+            sum
+          end
+        end
+      end
+
+      def rank(goal_pattern)
+        separated_goal_pattern = separate(goal_pattern)
+        all_separated_patterns.inject({}) do |ret, separated_pattern|
+          ret[separated_pattern.join] = score(separated_pattern, separated_goal_pattern)
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb

@@ -6,22 +6,31 @@
 module TwitterCldr
   module Tokenizers
     class DateTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
+      TOKEN_SPLITTER_REGEX = /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|\#\{[^\}]+\})/
+      TOKEN_TYPE_REGEXES = {
+        :composite => { :regex => /^\#\{[^\}]+\}/, :content => /^\#\{([^\}]+)\}/, :priority => 1 },
+        :pattern   => { :regex => /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/, :priority => 2 },
+        :plaintext => { :regex => //, :priority => 3 }
+      }
+
       def initialize(options = {})
         super(options)
-        @token_splitter_regex = /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|\#\{[^\}]+\})/
 
-        @token_type_regexes = [
-            { :type => :composite, :regex => /^\#\{[^\}]+\}/, :content => /^\#\{([^\}]+)\}/ },
-            { :type => :pattern,   :regex => /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/ },
-            { :type => :plaintext, :regex => // }
-        ]
+        @token_splitter_regexes = {
+          :else => TOKEN_SPLITTER_REGEX
+        }
+
+        @token_type_regexes = {
+          :else => TOKEN_TYPE_REGEXES
+        }
 
         @paths = {
-            :default => [:formats, :date, :default],
-            :full    => [:formats, :date, :full],
-            :long    => [:formats, :date, :long],
-            :medium  => [:formats, :date, :medium],
-            :short   => [:formats, :date, :short]
+          :default    => [:formats, :date, :default],
+          :full       => [:formats, :date, :full],
+          :long       => [:formats, :date, :long],
+          :medium     => [:formats, :date, :medium],
+          :short      => [:formats, :date, :short],
+          :additional => [:additional_formats]
         }
       end
 

data/lib/twitter_cldr/tokenizers/calendars/datetime_tokenizer.rb

@@ -8,38 +8,70 @@ module TwitterCldr
     class DateTimeTokenizer < Base
       attr_reader :placeholders, :calendar_type
 
-      VALID_TYPES = [:default, :full, :long, :medium, :short]
+      VALID_TYPES = [:default, :full, :long, :medium, :short, :additional]
 
       def initialize(options = {})
         @calendar_type = options[:calendar_type] || TwitterCldr::DEFAULT_CALENDAR_TYPE
+        @type = options[:type] || :default
+        @format = options[:format]
+
+        @token_splitter_regexes = {
+          :additional => Regexp.union(
+            DateTokenizer::TOKEN_SPLITTER_REGEX,
+            TimeTokenizer::TOKEN_SPLITTER_REGEX
+          ),
+          :else => //
+        }
 
-        @token_splitter_regex = //
-        @token_type_regexes   = [{ :type => :plaintext, :regex => // }]
+        @token_type_regexes = {
+          :additional => merge_token_type_regexes(DateTokenizer::TOKEN_TYPE_REGEXES, TimeTokenizer::TOKEN_TYPE_REGEXES),
+          :else => {
+            :plaintext => { :regex => // }
+          }
+        }
 
         @base_path = [:calendars]
 
         @paths = {
-            :default => [:formats, :datetime, :default],
-            :full    => [:formats, :datetime, :full],
-            :long    => [:formats, :datetime, :long],
-            :medium  => [:formats, :datetime, :medium],
-            :short   => [:formats, :datetime, :short]
+          :default    => [:formats, :datetime, :default],
+          :full       => [:formats, :datetime, :full],
+          :long       => [:formats, :datetime, :long],
+          :medium     => [:formats, :datetime, :medium],
+          :short      => [:formats, :datetime, :short],
+          :additional => [:additional_formats]
         }
 
         super(options)
       end
 
       def tokens(options = {})
-        type = options[:type] || :default
-        tokens_for(full_path_for(paths[type]), type)
+        @type = options[:type] || @type || :default
+        @format = options[:format] || @format
+        tokens_for(full_path_for(paths[@type]))
       end
 
       def calendar
         @resource[:calendars][@calendar_type]
       end
 
+      def additional_format_selector
+        @additional_format_selector ||= AdditionalDateFormatSelector.new(
+          @resource[:calendars][@calendar_type][:additional_formats]
+        )
+      end
+
       protected
 
+      def merge_token_type_regexes(first, second)
+        TwitterCldr::Utils.deep_merge_hash(first, second) do |left, right|
+          if right.is_a?(Regexp) && left.is_a?(Regexp)
+            Regexp.union(left, right)
+          else
+            left
+          end
+        end
+      end
+
       def full_path_for(path, calendar_type = @calendar_type)
         @base_path + [calendar_type] + path
       end
@@ -71,13 +103,18 @@ module TwitterCldr
 
       def init_placeholders
         @placeholders = [
-            { :name => :time, :object => TwitterCldr::Tokenizers::TimeTokenizer.new(:locale => @locale, :calendar_type => @calendar_type) },
-            { :name => :date, :object => TwitterCldr::Tokenizers::DateTokenizer.new(:locale => @locale, :calendar_type => @calendar_type) }
+          { :name => :time, :object => TwitterCldr::Tokenizers::TimeTokenizer.new(:locale => @locale, :calendar_type => @calendar_type) },
+          { :name => :date, :object => TwitterCldr::Tokenizers::DateTokenizer.new(:locale => @locale, :calendar_type => @calendar_type) }
         ]
       end
 
       def pattern_for(resource)
-        resource.is_a?(Hash) ? resource[:pattern] : resource
+        case type
+          when :additional
+            resource[additional_format_selector.find_closest(format).to_sym]
+          else
+            resource.is_a?(Hash) ? resource[:pattern] : resource
+        end
       end
     end
   end

data/lib/twitter_cldr/tokenizers/calendars/time_tokenizer.rb

@@ -6,21 +6,30 @@
 module TwitterCldr
   module Tokenizers
     class TimeTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
+      TOKEN_SPLITTER_REGEX = /(\'[\w\s-]+\'|a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/
+      TOKEN_TYPE_REGEXES = {
+        :pattern   => { :regex => /^(a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/, :priority => 1 },
+        :plaintext => { :regex => //, :priority => 2 }
+      }
+
       def initialize(options = {})
         super(options)
-        @token_splitter_regex = /(\'[\w\s-]+\'|a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/
 
-        @token_type_regexes = [
-            { :type => :pattern,   :regex => /^(a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/ },
-            { :type => :plaintext, :regex => // }
-        ]
+        @token_splitter_regexes = {
+          :else => TOKEN_SPLITTER_REGEX
+        }
+
+        @token_type_regexes = {
+          :else => TOKEN_TYPE_REGEXES
+        }
 
         @paths = {
-            :default => [:formats, :time, :default],
-            :full    => [:formats, :time, :full],
-            :long    => [:formats, :time, :long],
-            :medium  => [:formats, :time, :medium],
-            :short   => [:formats, :time, :short]
+          :default    => [:formats, :time, :default],
+          :full       => [:formats, :time, :full],
+          :long       => [:formats, :time, :long],
+          :medium     => [:formats, :time, :medium],
+          :short      => [:formats, :time, :short],
+          :additional => [:additional_formats]
         }
       end
 

data/lib/twitter_cldr/tokenizers/calendars/timespan_tokenizer.rb

@@ -15,43 +15,43 @@ module TwitterCldr
         @token_splitter_regex = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/ # creates spaces
 
         @token_type_regexes = [
-            { :type => :pattern, :regex => /[0?#,\.]*/ }, # splits token at right places
-            { :type => :plaintext, :regex => // }
+          { :type => :pattern, :regex => /[0?#,\.]*/ }, # splits token at right places
+          { :type => :plaintext, :regex => // }
         ]
 
         @base_path = [:units]
 
         @paths = {
-            :ago => {
-                :default => :'hour-past',
-                :second  => :'second-past',
-                :minute  => :'minute-past',
-                :hour    => :'hour-past',
-                :day     => :'day-past',
-                :week    => :'week-past',
-                :month   => :'month-past',
-                :year    => :'year-past'
-            },
-            :until => {
-                :default => :'hour-future',
-                :second  => :'second-future',
-                :minute  => :'minute-future',
-                :hour    => :'hour-future',
-                :day     => :'day-future',
-                :week    => :'week-future',
-                :month   => :'month-future',
-                :year    => :'year-future'
-            },
-            :none => {
-                :default => :second,
-                :second  => :second,
-                :minute  => :minute,
-                :hour    => :hour,
-                :day     => :day,
-                :week    => :week,
-                :month   => :month,
-                :year    => :year
-            }
+          :ago => {
+            :default => :'hour-past',
+            :second  => :'second-past',
+            :minute  => :'minute-past',
+            :hour    => :'hour-past',
+            :day     => :'day-past',
+            :week    => :'week-past',
+            :month   => :'month-past',
+            :year    => :'year-past'
+          },
+          :until => {
+            :default => :'hour-future',
+            :second  => :'second-future',
+            :minute  => :'minute-future',
+            :hour    => :'hour-future',
+            :day     => :'day-future',
+            :week    => :'week-future',
+            :month   => :'month-future',
+            :year    => :'year-future'
+          },
+          :none => {
+            :default => :second,
+            :second  => :second,
+            :minute  => :minute,
+            :hour    => :hour,
+            :day     => :day,
+            :week    => :week,
+            :month   => :month,
+            :year    => :year
+          }
         }
       end