twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/shared/code_point.rb

@@ -24,24 +24,54 @@ module TwitterCldr
       :simple_titlecase_map
     ]
 
-    CodePoint = Struct.new(*CODE_POINT_FIELDS) do
+    class CodePoint
       DECOMPOSITION_DATA_INDEX = 5
-
       DECOMPOSITION_REGEX = /^(?:<(.+)>\s+)?(.+)?$/
 
-      attr_accessor :compatibility_decomposition_tag
+      INDICES = [
+        :category, :bidi_class, :bidi_mirrored
+      ]
+
+      PROPERTIES = [
+        :sentence_break, :line_break, :word_break
+      ]
+
+      attr_reader :fields
 
-      def initialize(*)
-        super
+      CODE_POINT_FIELDS.each_with_index do |field, idx|
+        unless field == :decomposition
+          define_method field do
+            fields[idx]
+          end
+        end
+      end
 
-        if decomposition =~ DECOMPOSITION_REGEX
-          self.compatibility_decomposition_tag = $1
-          self.decomposition = $2 && $2.split.map(&:hex)
-        else
-          raise ArgumentError, "decomposition #{decomposition.inspect} has invalid format"
+      def decomposition
+        @decomposition ||= begin
+          decomp = fields[DECOMPOSITION_DATA_INDEX]
+          if decomp =~ DECOMPOSITION_REGEX
+            $2 && $2.split.map(&:hex)
+          else
+            raise ArgumentError, "decomposition #{decomp.inspect} has invalid format"
+          end
+        end
+      end
+
+      def compatibility_decomposition_tag
+        @compat_decomp_tag ||= begin
+          decomp = fields[DECOMPOSITION_DATA_INDEX]
+          if decomp =~ DECOMPOSITION_REGEX
+            $1
+          else
+            raise ArgumentError, "decomposition #{decomp.inspect} has invalid format"
+          end
         end
       end
 
+      def initialize(fields)
+        @fields = fields
+      end
+
       def compatibility_decomposition?
         !!compatibility_decomposition_tag
       end
@@ -65,7 +95,36 @@ module TwitterCldr
             block_data      = TwitterCldr.get_resource(:unicode_data, :blocks, target.first)
             code_point_data = block_data.fetch(code_point) { |cp| get_range_start(cp, block_data) }
 
-            CodePoint.new(*code_point_data) if code_point_data
+            CodePoint.new(code_point_data) if code_point_data
+          end
+        end
+
+        # Methods that return a list of code points for the given property name.
+        INDICES.each do |index_name|
+          define_method :"code_points_for_#{index_name}" do |value|
+            get_index(index_name)[value]
+          end
+        end
+
+        PROPERTIES.each do |property_name|
+          define_method :"code_points_for_#{property_name}" do |value|
+            get_property_data(property_name)[value]
+          end
+        end
+
+        # Search for code points wherein at least one property value contains prop_value.
+        # For example, if prop_value is set to :Zs, this method will return all code
+        # points that are considered spaces. If prop value is simply :Z, this method
+        # will return all code points who have a property value that contains :Z, i.e.
+        # spaces as well as line separators (:Zl) and paragraph separators (:Zp).
+        def code_points_for_property_value(prop_value)
+          index_key_cache[prop_value] ||= index_keys.inject([]) do |ret, (index_key, index_names)|
+            if index_key.to_s.include?(prop_value.to_s)
+              index_names.each do |index_name|
+                ret += get_index(index_name)[index_key]
+              end
+            end
+            ret
           end
         end
 
@@ -95,10 +154,6 @@ module TwitterCldr
           end
         end
 
-        def hangul_type_cache
-          @hangul_type_cache ||= {}
-        end
-
         def excluded_from_composition?(code_point)
           composition_exclusion_cache[code_point] ||=
             composition_exclusions.any? { |exclusion| exclusion.include?(code_point) }
@@ -106,6 +161,38 @@ module TwitterCldr
 
         private
 
+        def index_key_cache
+          @index_key_cache ||= {}
+        end
+
+        def index_keys
+          @index_keys ||= TwitterCldr.get_resource(:unicode_data, :indices, "keys")
+        end
+
+        def get_index(index_name)
+          index_cache[index_name] ||= TwitterCldr.get_resource(
+            :unicode_data, :indices, index_name
+          )
+        end
+
+        def get_property_data(property_name)
+          property_data_cache[property_name] ||= TwitterCldr.get_resource(
+            :unicode_data, :properties, property_name
+          )
+        end
+
+        def index_cache
+          @index_cache ||= {}
+        end
+
+        def property_data_cache
+          @property_data_cache ||= {}
+        end
+
+        def hangul_type_cache
+          @hangul_type_cache ||= {}
+        end
+
         def code_point_cache
           @code_point_cache ||= {}
         end
@@ -132,7 +219,7 @@ module TwitterCldr
         end
 
         def blocks
-          TwitterCldr.get_resource(:unicode_data, :blocks)
+          @blocks ||= TwitterCldr.get_resource(:unicode_data, :blocks)
         end
 
         # Check if block constitutes a range. The code point beginning a range will have a name enclosed in <>, ending with 'First'

data/lib/twitter_cldr/shared/numbering_system.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+
+    class UnsupportedNumberingSystemError < StandardError; end
+
+    class NumberingSystem
+
+      class << self
+        def for_name(name)
+          system_cache[name] ||= begin
+            if system = resource[name.to_sym]
+              if system[:type] != "numeric"
+                raise UnsupportedNumberingSystemError.new("#{system[:type]} numbering systems not supported.")
+              else
+                new(system[:name], system[:digits])
+              end
+            end
+          end
+        end
+
+        protected
+
+        def system_cache
+          @system_cache ||= {}
+        end
+
+        def resource
+          @resource ||= TwitterCldr.get_resource(:shared, :numbering_systems)[:numbering_systems]
+        end
+      end
+
+      attr_reader :name, :digits
+
+      def initialize(name, digits)
+        @name = name
+        @digits = split_digits(digits)
+      end
+
+      def transliterate(number)
+        number.to_s.gsub(/\d/) do |digit|
+          digits[digit.to_i]
+        end
+      end
+
+      protected
+
+      def split_digits(str)
+        str.unpack("U*").map { |digit| [digit].pack("U*") }
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/shared/territories.rb

@@ -29,7 +29,7 @@ module TwitterCldr
         # codes" (UN M.49; for example, 014 for Eastern Africa and 419 for Latin
         # America).
         def from_territory_code_for_locale(territory_code, locale = TwitterCldr.locale)
-          get_resource(locale)[:territories][TwitterCldr::Utils::Territories.normalize_territory_code(territory_code)]
+          get_resource(locale)[:territories][normalize_territory_code(territory_code)]
         rescue
           nil
         end
@@ -48,6 +48,48 @@ module TwitterCldr
           nil
         end
 
+        # Normalizes a territory code to a symbol.
+        #
+        # 1) Converts to string.
+        # 2) Downcases.
+        # 3) Symbolizes.
+        #
+        # The downcasing is to convert ISO 3166-1 alpha-2 codes,
+        # used (upper-case) for territories in CLDR, to be lowercase, to be
+        # consistent with how territory codes are surfaced in TwitterCLDR
+        # methods relating to phone and postal codes.
+        def normalize_territory_code(territory_code)
+          return if territory_code.nil?
+          territory_code.to_s.downcase.gsub(/^0+/, '').to_sym
+        end
+
+        # Normalizes each key in the 'arg' hash or constituent hashes as
+        # if it were a territory code.
+        #
+        # In addition, removes entries in hashes where the key begins with a digit.
+        # Because of the way the twitter-cldr-rb YAML resource pipeline works,
+        # these three-digit codes get mangled (e.g. interpreted as octal then
+        # reinterpreted out in decimal), and translations for UN three-digit
+        # area codes cannot be trusted.
+        def deep_normalize_territory_code_keys(arg)
+          case arg
+            when Array
+              arg.map { |elem| deep_normalize_territory_code_keys(elem) }
+            when Hash
+              normalized = arg.inject({}) do |carry, (key, value)|
+                normalized_key = normalize_territory_code(key)
+                carry[normalized_key] = deep_normalize_territory_code_keys(value)
+                carry
+              end
+              normalized.delete_if do |key, _|
+                key.to_s =~ /^\d+$/
+              end
+              normalized
+            else
+              arg
+          end
+        end
+
         protected
 
         def get_resource(locale)

data/lib/twitter_cldr/shared/unicode_regex.rb

@@ -0,0 +1,81 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+    class UnicodeRegex
+
+      class << self
+
+        def compile(str, modifiers = "", symbol_table = nil)
+          new(
+            parser.parse(tokenizer.tokenize(str), {
+              :symbol_table => symbol_table
+            }), modifiers
+          )
+        end
+
+        # All unicode characters
+        def all_unicode
+          @all_unicode ||= TwitterCldr::Utils::RangeSet.new(
+            [0..0x10FFFF]
+          )
+        end
+
+        # A few <control> characters (i.e. 2..7) and public/private surrogates (i.e. 55296..57343).
+        # These don't play nicely with Ruby's regular expression engine, and I think we
+        # can safely disregard them.
+        def invalid_regexp_chars
+          @invalid_regexp_chars ||= TwitterCldr::Utils::RangeSet.new(
+            [2..7, 55296..57343]
+          )
+        end
+
+        def valid_regexp_chars
+          @valid_regexp_chars ||= all_unicode.subtract(invalid_regexp_chars)
+        end
+
+        private
+
+        def tokenizer
+          @tokenizer ||= TwitterCldr::Tokenizers::UnicodeRegexTokenizer.new
+        end
+
+        def parser
+          @parser ||= TwitterCldr::Parsers::UnicodeRegexParser.new
+        end
+
+      end
+
+      extend Forwardable
+      def_delegator :to_regexp, :match
+      def_delegator :to_regexp, :=~
+
+      attr_reader :elements, :modifiers
+
+      def initialize(elements, modifiers = nil)
+        @elements = elements
+        @modifiers = nil
+      end
+
+      def to_regexp
+        if RUBY_VERSION <= "1.8.7"
+          begin
+            Oniguruma::ORegexp.new(to_regexp_str, modifiers)
+          rescue NameError
+            raise "Unicode regular expressions require the Oniguruma gem when using Ruby 1.8. Please install, require, and retry."
+          end
+        else
+          @regexp ||= Regexp.new(to_regexp_str, modifiers)
+        end
+      end
+
+      def to_regexp_str
+        @regexp_str ||= elements.map(&:to_regexp_str).join
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/shared.rb

@@ -5,15 +5,19 @@
 
 module TwitterCldr
   module Shared
-    autoload :Calendar,      'twitter_cldr/shared/calendar'
-    autoload :CodePoint,     'twitter_cldr/shared/code_point'
-    autoload :Currencies,    'twitter_cldr/shared/currencies'
-    autoload :LanguageCodes, 'twitter_cldr/shared/language_codes'
-    autoload :Languages,     'twitter_cldr/shared/languages'
-    autoload :Numbers,       'twitter_cldr/shared/numbers'
-    autoload :PhoneCodes,    'twitter_cldr/shared/phone_codes'
-    autoload :PostalCodes,   'twitter_cldr/shared/postal_codes'
-    autoload :Bidi,          'twitter_cldr/shared/bidi'
+    autoload :Calendar,        'twitter_cldr/shared/calendar'
+    autoload :CodePoint,       'twitter_cldr/shared/code_point'
+    autoload :Currencies,      'twitter_cldr/shared/currencies'
+    autoload :LanguageCodes,   'twitter_cldr/shared/language_codes'
+    autoload :Languages,       'twitter_cldr/shared/languages'
+    autoload :Numbers,         'twitter_cldr/shared/numbers'
+    autoload :PhoneCodes,      'twitter_cldr/shared/phone_codes'
+    autoload :PostalCodes,     'twitter_cldr/shared/postal_codes'
+    autoload :Bidi,            'twitter_cldr/shared/bidi'
     autoload :Territories,     'twitter_cldr/shared/territories'
+    autoload :NumberingSystem, 'twitter_cldr/shared/numbering_system'
+    autoload :Casefolder,      'twitter_cldr/shared/casefolder'
+    autoload :UnicodeRegex,    'twitter_cldr/shared/unicode_regex'
+    autoload :BreakIterator,   'twitter_cldr/shared/break_iterator'
   end
 end

data/lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb

@@ -0,0 +1,77 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class DateTimeTokenizer
+
+      attr_reader :data_reader
+
+      def initialize(data_reader)
+        @data_reader = data_reader
+      end
+
+      def tokenize(pattern)
+        expand_tokens(
+          PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
+        )
+      end
+
+      # Tokenizes mixed date and time pattern strings,
+      # used to tokenize the additional date format patterns.
+      def full_tokenize(pattern)
+        PatternTokenizer.new(data_reader, full_tokenizer).tokenize(pattern)
+      end
+
+      protected
+
+      def expand_tokens(tokens)
+        tokens.inject([]) do |ret, token|
+          ret + case token.type
+            when :date
+              expand_date(token)
+            when :time
+              expand_time(token)
+            else
+              [token]
+          end
+        end
+      end
+
+      def expand_date(token)
+        date_reader = data_reader.date_reader
+        date_reader.tokenizer.tokenize(date_reader.pattern)
+      end
+
+      def expand_time(token)
+        time_reader = data_reader.time_reader
+        time_reader.tokenizer.tokenize(time_reader.pattern)
+      end
+
+      def full_tokenizer
+        @@full_tokenizer ||= begin
+          new_tok = Tokenizer.union(
+            data_reader.date_reader.tokenizer.tokenizer,
+            data_reader.time_reader.tokenizer.tokenizer
+          ) do |recognizer|
+            recognizer.token_type != :plaintext
+          end
+
+          new_tok.recognizers << TokenRecognizer.new(:plaintext, //)
+          new_tok
+        end
+      end
+
+      def tokenizer
+        @tokenizer ||= Tokenizer.new([
+          TokenRecognizer.new(:date, /\{\{date\}\}/),
+          TokenRecognizer.new(:time, /\{\{time\}\}/),
+          TokenRecognizer.new(:plaintext, //)
+        ])
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb

@@ -5,41 +5,26 @@
 
 module TwitterCldr
   module Tokenizers
-    class DateTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
-      TOKEN_SPLITTER_REGEX = /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|\#\{[^\}]+\})/
-      TOKEN_TYPE_REGEXES = {
-        :composite => { :regex => /^\#\{[^\}]+\}/, :content => /^\#\{([^\}]+)\}/, :priority => 1 },
-        :pattern   => { :regex => /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/, :priority => 2 },
-        :plaintext => { :regex => //, :priority => 3 }
-      }
+    class DateTokenizer
 
-      def initialize(options = {})
-        super(options)
+      attr_reader :data_reader
 
-        @token_splitter_regexes = {
-          :else => TOKEN_SPLITTER_REGEX
-        }
-
-        @token_type_regexes = {
-          :else => TOKEN_TYPE_REGEXES
-        }
-
-        @paths = {
-          :default    => [:formats, :date, :default],
-          :full       => [:formats, :date, :full],
-          :long       => [:formats, :date, :long],
-          :medium     => [:formats, :date, :medium],
-          :short      => [:formats, :date, :short],
-          :additional => [:additional_formats]
-        }
+      def initialize(data_reader)
+        @data_reader = data_reader
       end
 
-      protected
+      def tokenize(pattern)
+        PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
+      end
 
-      # must override this because DateTimeTokenizer will set them otherwise
-      def init_placeholders
-        @placeholders = {}
+      def tokenizer
+        @tokenizer ||= Tokenizer.new([
+          TokenRecognizer.new(:composite, /^\#\{[^\}]+\}/, /^\#\{([^\}]+)\}/),
+          TokenRecognizer.new(:pattern, /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5})/),
+          TokenRecognizer.new(:plaintext, //)
+        ], /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|\#\{[^\}]+\})/)
       end
+
     end
   end
 end

data/lib/twitter_cldr/tokenizers/calendars/time_tokenizer.rb

@@ -5,40 +5,25 @@
 
 module TwitterCldr
   module Tokenizers
-    class TimeTokenizer < TwitterCldr::Tokenizers::DateTimeTokenizer
-      TOKEN_SPLITTER_REGEX = /(\'[\w\s-]+\'|a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/
-      TOKEN_TYPE_REGEXES = {
-        :pattern   => { :regex => /^(a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/, :priority => 1 },
-        :plaintext => { :regex => //, :priority => 2 }
-      }
+    class TimeTokenizer
 
-      def initialize(options = {})
-        super(options)
+      attr_reader :data_reader
 
-        @token_splitter_regexes = {
-          :else => TOKEN_SPLITTER_REGEX
-        }
-
-        @token_type_regexes = {
-          :else => TOKEN_TYPE_REGEXES
-        }
-
-        @paths = {
-          :default    => [:formats, :time, :default],
-          :full       => [:formats, :time, :full],
-          :long       => [:formats, :time, :long],
-          :medium     => [:formats, :time, :medium],
-          :short      => [:formats, :time, :short],
-          :additional => [:additional_formats]
-        }
+      def initialize(data_reader)
+        @data_reader = data_reader
       end
 
-      protected
+      def tokenize(pattern)
+        PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
+      end
 
-      # must override this because DateTimeTokenizer will set them otherwise
-      def init_placeholders
-        @placeholders = {}
+      def tokenizer
+        @tokenizer ||= Tokenizer.new([
+          TokenRecognizer.new(:pattern, /^(a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/),
+          TokenRecognizer.new(:plaintext, //)
+        ], /(\'[\w\s-]+\'|a{1}|h{1,2}|H{1,2}|K{1,2}|k{1,2}|m{1,2}|s{1,2}|S+|z{1,4}|Z{1,4})/)
       end
+
     end
   end
 end

data/lib/twitter_cldr/tokenizers/calendars/timespan_tokenizer.rb

@@ -5,103 +5,27 @@
 
 module TwitterCldr
   module Tokenizers
-    class TimespanTokenizer < Base
+    class TimespanTokenizer
 
-      VALID_UNITS = [:second, :minute, :hour, :day, :week, :month, :year]
+      attr_reader :data_reader
 
-      def initialize(options = {})
-        super(options)
-
-        @token_splitter_regex = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/ # creates spaces
-
-        @token_type_regexes = [
-          { :type => :pattern, :regex => /[0?#,\.]*/ }, # splits token at right places
-          { :type => :plaintext, :regex => // }
-        ]
-
-        @base_path = [:units]
-
-        @paths = {
-          :ago => {
-            :default => :'hour-past',
-            :second  => :'second-past',
-            :minute  => :'minute-past',
-            :hour    => :'hour-past',
-            :day     => :'day-past',
-            :week    => :'week-past',
-            :month   => :'month-past',
-            :year    => :'year-past'
-          },
-          :until => {
-            :default => :'hour-future',
-            :second  => :'second-future',
-            :minute  => :'minute-future',
-            :hour    => :'hour-future',
-            :day     => :'day-future',
-            :week    => :'week-future',
-            :month   => :'month-future',
-            :year    => :'year-future'
-          },
-          :none => {
-            :default => :second,
-            :second  => :second,
-            :minute  => :minute,
-            :hour    => :hour,
-            :day     => :day,
-            :week    => :week,
-            :month   => :month,
-            :year    => :year
-          }
-        }
-      end
-
-      def tokens(options = {})
-        # tokens_with_placeholders_for(full_path(:none, :second, :short, :two))
-        path = full_path(options[:direction], options[:unit], options[:type])
-        pluralization = options[:rule] || TwitterCldr::Formatters::Plurals::Rules.rule_for(options[:number], @locale)
-        available = traverse(path)
-
-        case pluralization # sometimes the plural rule will return ":one" when the resource only contains a path with "1"
-          when :zero
-            pluralization = 0 if available.include?(0)
-          when :one
-            pluralization = 1 if available.include?(1)
-          when :two
-            pluralization = 2 if available.include?(2)
-        end
-
-        if available.include?(pluralization)
-          path << pluralization
-        else
-          return [] unless available.keys.first
-          path << available.keys.first
-        end
-
-        tokens_with_placeholders_for(path)
+      def initialize(data_reader)
+        @data_reader = data_reader
       end
 
-      def all_types_for(unit, direction)
-        traverse(@base_path + [@paths[direction][unit]]).keys
+      def tokenize(pattern)
+        PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
       end
 
       protected
 
-      def token_cache
-        @token_cache ||= {}
-      end
-
-      def full_path(direction, unit, type)
-        @base_path + [@paths[direction][unit], type]
+      def tokenizer
+        @tokenizer ||= Tokenizer.new([
+          TokenRecognizer.new(:pattern, /\{?[0?#,\.]+\}?/),
+          TokenRecognizer.new(:plaintext, //)
+        ], /([^0*#,\.\{\}]*)(\{?[0#,\.]+\}?)([^0*#,\.\{\}]*)$/)
       end
 
-      def init_resources
-        @resource = TwitterCldr.get_locale_resource(@locale, :units)[TwitterCldr.convert_locale(@locale)]
-      end
-
-      def pattern_for(resource)
-        # can't go any deeper, so return original pattern (which should NOT be a hash, by the way)
-        resource
-      end
     end
   end
 end