twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb

@@ -5,88 +5,33 @@
 
 module TwitterCldr
   module Tokenizers
-    class NumberTokenizer < Base
-      ABBREVIATED_MIN_POWER = 3
-      ABBREVIATED_MAX_POWER = 14
+    class NumberTokenizer
 
-      VALID_TYPES = [:decimal, :percent, :currency, :short_decimal, :long_decimal]
-      TOKEN_SPLITTER_REGEX = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/
-      TOKEN_TYPE_REGEXES = {
-        :pattern => { :regex => /[0?#,\.]*/, :priority => 1 },
-        :plaintext => { :regex => //, :priority => 2 }
-      }
+      attr_reader :data_reader
 
-      def initialize(options = {})
-        super(options)
-
-        @token_splitter_regexes = {
-          :else => TOKEN_SPLITTER_REGEX
-        }
-
-        @token_type_regexes = {
-          :else => TOKEN_TYPE_REGEXES
-        }
-
-        @base_path   = [:numbers, :formats]
-        @symbol_path = [:numbers, :symbols]
-
-        @paths = {
-          :default       => [:decimal, :patterns],
-          :decimal       => [:decimal, :patterns],
-          :long_decimal  => [:decimal, :patterns, :long],
-          :short_decimal => [:decimal, :patterns, :short],
-          :currency      => [:currency, :patterns],
-          :percent       => [:percent, :patterns]
-        }
-      end
-
-      def tokens(options = {})
-        @type = options[:type] || :default
-        @format = options[:format] || :default
-
-        path = full_path
-        positive, negative = traverse(path).to_s.split(/;/)
-        sign = options[:sign] || :positive
-
-        pattern = case sign
-          when :negative
-            if negative
-              "#{symbols[:minus] || '-'}#{negative}"
-            else
-              "#{symbols[:minus] || '-'}#{positive}"
-            end
-          else
-            positive
-        end
-
-        tokens_for_pattern(pattern, path, [sign])
+      def initialize(data_reader)
+        @data_reader = data_reader
       end
 
-      def symbols
-        traverse(@symbol_path)
-      end
+      def tokenize(pattern)
+        tokens = PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
 
-      def valid_types
-        VALID_TYPES.select do |type|
-          result = traverse(@base_path + @paths[type])
-          !!result ? result.size > 0 : result
+        if tokens.first.value == ""
+          tokens[1..-1]
+        else
+          tokens
         end
       end
 
-      protected
+      private
 
-      def full_path
-        @base_path + paths[@type] + [@format]
+      def tokenizer
+        @tokenizer ||= Tokenizer.new([
+          TokenRecognizer.new(:pattern, /[0?#,\.]+/),
+          TokenRecognizer.new(:plaintext, //),
+        ], /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/, false)
       end
 
-      def init_resources
-        @resource = TwitterCldr.get_locale_resource(@locale, :numbers)[TwitterCldr.convert_locale(@locale)]
-      end
-
-      def pattern_for(resource)
-        # can't go any deeper, so return original pattern (which should NOT be a hash, by the way)
-        resource
-      end
     end
   end
 end

data/lib/twitter_cldr/tokenizers/numbers/rbnf_tokenizer.rb

@@ -0,0 +1,53 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class RbnfTokenizer
+
+      def tokenize(pattern)
+        PatternTokenizer.new(nil, tokenizer).tokenize(pattern)
+      end
+
+      private
+
+      def tokenizer
+        # i.e. %spellout-numbering, %%2d-year
+        rule_regex = if RUBY_VERSION <= "1.8.7"
+          /%%?[\w\-]+/u
+        else
+          Regexp.new("%%?[[:word:]\-]+")
+        end
+
+        recognizers = [
+          # special rule descriptors
+          TokenRecognizer.new(:negative, /-x/),
+          TokenRecognizer.new(:improper_fraction, /x\.x/),
+          TokenRecognizer.new(:proper_fraction, /0\.x/),
+          TokenRecognizer.new(:master, /x\.0/),
+
+          # normal rule descriptors
+          TokenRecognizer.new(:equals, /=/),
+          TokenRecognizer.new(:rule, rule_regex),
+          TokenRecognizer.new(:right_arrow, />/),
+          TokenRecognizer.new(:left_arrow, /</),
+          TokenRecognizer.new(:open_bracket, /\[/),
+          TokenRecognizer.new(:close_bracket, /\]/),
+          TokenRecognizer.new(:decimal, /[0#][0#,\.]+/),
+
+          # ending
+          TokenRecognizer.new(:semicolon, /;/),
+        ]
+
+        @tokenizer ||= Tokenizer.new(
+          recognizers + [
+            TokenRecognizer.new(:plaintext, //)  # catch-all
+          ]
+        )
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/pattern_tokenizer.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class PatternTokenizer
+
+      attr_reader :data_reader, :tokenizer
+
+      def initialize(data_reader, tokenizer)
+        @data_reader = data_reader
+        @tokenizer = tokenizer
+      end
+
+      def tokenize(pattern)
+        tokenizer.tokenize(expand(pattern))
+      end
+
+      private
+
+      def expand(pattern)
+        if pattern.is_a?(Symbol)
+          # symbols mean another path was given
+          path = pattern.to_s.split(".").map(&:to_sym)
+          data = data_reader.pattern_at_path(path)
+          next_pattern = data.is_a?(Hash) ? data[:pattern] : data
+          expand_pattern(next_pattern)
+        elsif pattern.is_a?(Hash)
+          pattern.inject({}) do |ret, (key, val)|
+            ret[key] = expand(val)
+            ret
+          end
+        else
+          pattern
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/segmentation/segmentation_tokenizer.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class SegmentationTokenizer
+
+      def tokenize(pattern)
+        # according to the spec, whitespace should be ignored
+        tokenizer.tokenize(pattern).reject do |token|
+          token.value.strip.empty?
+        end
+      end
+
+      private
+
+      def tokenizer
+        @tokenizer ||= begin
+          recognizers = [
+            TokenRecognizer.new(:break, /\303\267/u) do |val|     # ÷ character
+              val.strip
+            end,
+
+            TokenRecognizer.new(:no_break, /\303\227/u) do |val|  # × character
+              val.strip
+            end
+          ]
+
+          ur_tokenizer = UnicodeRegexTokenizer.new
+          ur_tokenizer.insert_before(:string, *recognizers)
+          ur_tokenizer
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/tokenizer.rb

@@ -0,0 +1,116 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+
+    class TokenRecognizer
+
+      attr_reader :token_type, :regex, :content, :cleaner
+
+      def initialize(token_type, regex, content = nil, &block)
+        @token_type = token_type
+        @regex = regex
+        @content = content
+        @cleaner = block
+      end
+
+      def recognizes?(text)
+        !!(text =~ regex)
+      end
+
+      def clean(val)
+        if cleaner
+          cleaner.call(val)
+        else
+          val
+        end
+      end
+
+    end
+
+    class Tokenizer
+
+      attr_reader :recognizers, :custom_splitter, :remove_empty_entries
+
+      def self.union(*tokenizers)
+        recognizers = tokenizers.inject([]) do |ret, tokenizer|
+          ret + tokenizer.recognizers.inject([]) do |recog_ret, recognizer|
+            if (block_given? && yield(recognizer)) || !block_given?
+              recog_ret << recognizer
+            end
+            recog_ret
+          end
+        end
+
+        splitter = if tokenizers.all?(&:custom_splitter)
+          Regexp.compile(
+            tokenizers.map do |tokenizer|
+              tokenizer.custom_splitter.source
+            end.join("|")
+          )
+        end
+
+        new(recognizers, splitter)
+      end
+
+      def initialize(recognizers, splitter = nil, remove_empty_entries = true)
+        @recognizers = recognizers
+        @custom_splitter = splitter
+        @remove_empty_entries = remove_empty_entries
+      end
+
+      def recognizer_at(token_type)
+        recognizers.find { |r| r.token_type == token_type }
+      end
+
+      def insert_before(token_type, *new_recognizers)
+        idx = recognizers.find_index { |rec| rec.token_type == token_type }
+        recognizers.insert(idx, *new_recognizers)
+        clear_splitter
+        nil
+      end
+
+      def tokenize(text)
+        text.split(splitter).inject([]) do |ret, token_text|
+          recognizer = recognizers.find do |recognizer|
+            recognizer.recognizes?(token_text)
+          end
+
+          if recognizer.token_type == :composite
+            content = token_text.match(recognizer.content)[1]
+            ret << CompositeToken.new(tokenize(content))
+          else
+            cleaned_text = recognizer.clean(token_text)
+
+            if (remove_empty_entries && cleaned_text.size > 0) || !remove_empty_entries
+              ret << Token.new(
+                :value => cleaned_text,
+                :type => recognizer.token_type
+              )
+            end
+          end
+
+          ret
+        end
+      end
+
+      private
+
+      def splitter
+        @splitter ||= (@custom_splitter || Regexp.new(
+          "(" + recognizers.map do |recognizer|
+            recognizer.regex.source
+          end.join("|") + ")"
+        ))
+      end
+
+      def clear_splitter
+        @splitter = nil
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers/unicode_regex/unicode_regex_tokenizer.rb

@@ -0,0 +1,52 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Tokenizers
+    class UnicodeRegexTokenizer
+
+      extend Forwardable
+      def_delegator :tokenizer, :insert_before
+
+      def tokenize(pattern)
+        tokenizer.tokenize(pattern)
+      end
+
+      private
+
+      def tokenizer
+        @tokenizer ||= begin
+          recognizers = [
+            # The variable name can contain letters and digits, but must start with a letter.
+            TokenRecognizer.new(:variable, /\$\w[\w\d]*/),
+            TokenRecognizer.new(:character_set, /\[:\w+:\]|\\p\{[\w=]+\}/),  # [:Lu:] or \p{Lu} or \p{Sentence_Break=CF}
+            TokenRecognizer.new(:negated_character_set, /\[:\^\w+:\]|\\P\{[\w=]+\}/),  #[:^Lu:] or \P{Lu}
+            TokenRecognizer.new(:unicode_char, /\\u\{?[a-fA-F0-9]{1,6}\}?/),
+            TokenRecognizer.new(:multichar_string, /\{\w+\}/u),
+
+            TokenRecognizer.new(:escaped_character, /\\./),
+            TokenRecognizer.new(:negate, /\^/),
+            TokenRecognizer.new(:ampersand, /&/),
+            TokenRecognizer.new(:pipe, /\|/),
+            TokenRecognizer.new(:dash, /-/),
+
+            # stuff that shouldn't be converted to codepoints
+            TokenRecognizer.new(:special_char, /\{\d,?\d?\}|[$?:{}()*+\.,\/\\]/),
+
+            TokenRecognizer.new(:open_bracket, /\[/),
+            TokenRecognizer.new(:close_bracket, /\]/),
+
+            TokenRecognizer.new(:string, //) do |val|
+              val == " " ? val : val.strip
+            end
+          ]
+
+          Tokenizer.new(recognizers)
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/tokenizers.rb

@@ -6,14 +6,18 @@
 module TwitterCldr
   module Tokenizers
     autoload :Base,                         'twitter_cldr/tokenizers/base'
-    autoload :CompositeToken,               'twitter_cldr/tokenizers/composite_token'
-    autoload :KeyPath,                      'twitter_cldr/tokenizers/key_path'
     autoload :Token,                        'twitter_cldr/tokenizers/token'
-    autoload :DateTimeTokenizer,            'twitter_cldr/tokenizers/calendars/datetime_tokenizer'
+    autoload :CompositeToken,               'twitter_cldr/tokenizers/composite_token'
+    autoload :Tokenizer,                    'twitter_cldr/tokenizers/tokenizer'
+    autoload :TokenRecognizer,              'twitter_cldr/tokenizers/tokenizer'
+    autoload :PatternTokenizer,             'twitter_cldr/tokenizers/pattern_tokenizer'
+    autoload :DateTimeTokenizer,            'twitter_cldr/tokenizers/calendars/date_time_tokenizer'
     autoload :DateTokenizer,                'twitter_cldr/tokenizers/calendars/date_tokenizer'
     autoload :TimeTokenizer,                'twitter_cldr/tokenizers/calendars/time_tokenizer'
     autoload :NumberTokenizer,              'twitter_cldr/tokenizers/numbers/number_tokenizer'
+    autoload :RbnfTokenizer,                'twitter_cldr/tokenizers/numbers/rbnf_tokenizer'
     autoload :TimespanTokenizer,            'twitter_cldr/tokenizers/calendars/timespan_tokenizer'
-    autoload :AdditionalDateFormatSelector, 'twitter_cldr/tokenizers/calendars/additional_date_format_selector'
+    autoload :UnicodeRegexTokenizer,        'twitter_cldr/tokenizers/unicode_regex/unicode_regex_tokenizer'
+    autoload :SegmentationTokenizer,        'twitter_cldr/tokenizers/segmentation/segmentation_tokenizer'
   end
 end

data/lib/twitter_cldr/utils/code_points.rb

@@ -26,7 +26,7 @@ module TwitterCldr
         end
 
         def from_string(str)
-          from_chars(str.chars.to_a)
+          str.unpack("U*")
         end
 
         def to_string(code_points)

data/lib/twitter_cldr/utils/range_set.rb

@@ -0,0 +1,242 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'set'
+
+module TwitterCldr
+  module Utils
+
+    # An integer set, implemented under the hood with ranges. The idea is
+    # that it's more efficient to store sequential data in ranges rather
+    # than as single elements. By definition, RangeSets contain no duplicates.
+    class RangeSet
+
+      attr_reader :ranges
+
+      class << self
+
+        def from_array(array)
+          new(rangify(array))
+        end
+
+        # Turns an array of integers into ranges. The "compress" option indicates
+        # wether or not to turn isolated elements into zero-length ranges or leave
+        # them as single elements.
+        #
+        # For example:
+        # rangify([1, 2, 4], false) returns [1..2, 4..4]
+        # rangify([1, 2, 4], true) returns [1..2, 4]
+        def rangify(list, compress = false)
+          last_item = nil
+
+          list.sort.inject([]) do |ret, item|
+            if last_item
+              diff = item - last_item
+
+              if diff > 0
+                if diff == 1
+                  ret[-1] << item
+                else
+                  ret << [item]
+                end
+
+                last_item = item
+              end
+            else
+              ret << [item]
+              last_item = item
+            end
+
+            ret
+          end.map do |sub_list|
+            if compress && sub_list.size == 1
+              sub_list.first
+            else
+              sub_list.first..sub_list.last
+            end
+          end
+        end
+
+      end
+
+      def initialize(ranges)
+        @ranges = ranges
+        flatten
+      end
+
+      def to_a(compress = false)
+        if compress
+          ranges.map do |range|
+            if range.first == range.last
+              range.first
+            else
+              range
+            end
+          end
+        else
+          ranges.dup
+        end
+      end
+
+      def to_full_a
+        ranges.inject([]) do |ret, range|
+          ret + range.to_a
+        end
+      end
+
+      def to_set
+        Set.new(to_full_a)
+      end
+
+      def include?(obj)
+        case obj
+          when Numeric
+            ranges.any? { |range| range.include?(obj) }
+          when Range
+            ranges.any? do |range|
+              range.first <= obj.first && range.last >= obj.last
+            end
+          else
+            false
+        end
+      end
+
+      def empty?
+        ranges.empty?
+      end
+
+      def union(range_set)
+        self.class.new(range_set.ranges + ranges)
+      end
+
+      def intersection(range_set)
+        new_ranges = []
+
+        range_set.ranges.each do |their_range|
+          ranges.each do |our_range|
+            if overlap?(their_range, our_range)
+              if intrsc = find_intersection(our_range, their_range)
+                new_ranges << intrsc
+              end
+            end
+          end
+        end
+
+        self.class.new(new_ranges)
+      end
+
+      def subtract(range_set)
+        return self if range_set.empty?
+        remaining = range_set.ranges.dup
+        current_ranges = ranges.dup
+        new_ranges = []
+
+        while their_range = remaining.shift
+          new_ranges = []
+
+          current_ranges.each do |our_range|
+            if overlap?(their_range, our_range)
+              new_ranges += find_subtraction(their_range, our_range)
+            else
+              new_ranges << our_range
+            end
+          end
+
+          current_ranges = new_ranges
+        end
+
+        self.class.new(new_ranges)
+      end
+
+      # symmetric difference (the union without the intersection)
+      # http://en.wikipedia.org/wiki/Symmetric_difference
+      def difference(range_set)
+        union(range_set).subtract(intersection(range_set))
+      end
+
+      private
+
+      def flatten
+        return if ranges.size <= 1
+
+        sorted_ranges = ranges.sort do |a, b|
+          if is_numeric_range?(a) && is_numeric_range?(b)
+            a.first <=> b.first
+          else
+            1
+          end
+        end
+
+        new_ranges = [sorted_ranges.first]
+
+        sorted_ranges.each do |range|
+          previous_range = new_ranges.pop
+
+          if adjacent?(previous_range, range) || overlap?(previous_range, range)
+            new_ranges.push(
+              [range.first, previous_range.first].min..[range.last, previous_range.last].max
+            )
+          else
+            new_ranges.push(previous_range)
+            new_ranges.push(range)
+          end
+        end
+
+        @ranges = new_ranges
+      end
+
+      # returns true if range1 and range2 are within 1 of each other
+      def adjacent?(range1, range2)
+        is_numeric_range?(range1) && is_numeric_range?(range2) &&
+          (range1.last == range2.first - 1 || range2.first == range1.last + 1)
+      end
+
+      def overlap?(range1, range2)
+        is_numeric_range?(range1) && is_numeric_range?(range2) && (
+          (range1.last >= range2.first && range1.last <= range2.last) ||
+          (range1.first >= range2.first && range1.first <= range2.last) ||
+          (range1.first <= range2.first && range1.last >= range2.last)
+        )
+      end
+
+      def find_intersection(range1, range2)
+        # range2 entirely contains range1
+        if range2.first <= range1.first && range1.last <= range2.last
+          range1.dup
+        elsif range1.last >= range2.first && range1.last <= range2.last
+          range2.first..range1.last
+        elsif range1.first >= range2.first && range1.first <= range2.last
+          range1.first..range2.last
+        elsif range1.first <= range2.first && range1.last >= range2.last
+          [range1.first, range2.first].max..[range1.last, range2.last].min
+        end
+      end
+
+      # subtracts range1 from range2 (range2 - range1)
+      def find_subtraction(range1, range2)
+        # case: range1 contains range2 entirely (also handles equal case)
+        result = if range1.first <= range2.first && range2.last <= range1.last
+          []
+        # case: range1 comes in the middle
+        elsif range2.first <= range1.first && range2.last >= range1.last
+          [range2.first..(range1.first - 1), (range1.last + 1)..range2.last]
+        # case: range1 trails
+        elsif range2.last >= range1.first && range1.last >= range2.last
+          [range2.first..(range1.first - 1)]
+        # case: range1 leads
+        elsif range1.last >= range2.first && range1.first <= range2.first
+          [(range1.last + 1)..range2.last]
+        end
+
+        result.reject { |r| r.first > r.last }
+      end
+
+      def is_numeric_range?(range)
+        range.first.is_a?(Numeric) && range.last.is_a?(Numeric)
+      end
+
+    end
+  end
+end