twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/localized/localized_object.rb

@@ -18,12 +18,12 @@ module TwitterCldr
         options = options.dup
         options[:locale] = @locale
 
-        @formatter = formatter_const.new(options) if formatter_const
+        # @formatter = formatter_const.new(options) if formatter_const
       end
 
-      def formatter_const
-        raise NotImplementedError
-      end
+      # def formatter_const
+      #   raise NotImplementedError
+      # end
 
       def self.localize(klass)
         klass.class_eval <<-LOCALIZE, __FILE__, __LINE__ + 1

data/lib/twitter_cldr/localized/localized_string.rb

@@ -16,16 +16,34 @@ module TwitterCldr
       # pluralization specification).
       #
       def %(args)
-        pluralized = args.is_a?(Hash) ? @formatter.format(@base_obj, args) : @base_obj
+        pluralized = args.is_a?(Hash) ? formatter.format(@base_obj, args) : @base_obj
         TwitterCldr::Utils.interpolate(pluralized, args)
       end
 
-      def formatter_const
-        TwitterCldr::Formatters::PluralFormatter
+      def normalize(options = {})
+        TwitterCldr::Normalization.normalize(@base_obj, options).localize(locale)
       end
 
-      def normalize(options = {})
-        TwitterCldr::Normalization.normalize(@base_obj, options).localize(@locale)
+      def casefold(options = {})
+        unless options.include?(:t)
+          # Turkish and azerbaijani use the dotless i and therefore have a few
+          # special casefolding rules. Note "az" is not actually supported yet.
+          options[:t] = [:tr, :az].include?(locale)
+        end
+
+        TwitterCldr::Shared::Casefolder.
+          casefold(@base_obj, options[:t]).
+          localize(locale)
+      end
+
+      def each_sentence
+        if block_given?
+          break_iterator.each_sentence(@base_obj) do |sentence|
+            yield sentence.localize
+          end
+        else
+          to_enum(__method__)
+        end
       end
 
       def code_points
@@ -41,8 +59,8 @@ module TwitterCldr
       end
 
       def to_f(options = {})
-        if TwitterCldr::Parsers::NumberParser.is_numeric?(@base_obj)
-          TwitterCldr::Parsers::NumberParser.new(@locale).try_parse(@base_obj, options) do |result|
+        if number_parser.class.is_numeric?(@base_obj)
+          number_parser.try_parse(@base_obj, options) do |result|
             result || @base_obj.to_f
           end
         else
@@ -94,6 +112,21 @@ module TwitterCldr
         to_bidi(options).reorder_visually!.to_s
       end
 
+      protected
+
+      def formatter
+        @formatter ||=
+          TwitterCldr::Formatters::PluralFormatter.for_locale(locale)
+      end
+
+      def break_iterator
+        @break_iterator ||= TwitterCldr::Shared::BreakIterator.new(locale)
+      end
+
+      def number_parser
+        @number_parser ||= TwitterCldr::Parsers::NumberParser.new(locale)
+      end
+
     end
 
   end

data/lib/twitter_cldr/localized/localized_time.rb

@@ -35,9 +35,16 @@ module TwitterCldr
         timezone_info.utc_to_local(@base_obj.utc)
       end
 
-      def formatter_const
-        TwitterCldr::Formatters::TimeFormatter
+      def data_reader_for(type)
+        TwitterCldr::DataReaders::TimeDataReader.new(locale, {
+          :calendar_type => calendar_type,
+          :type => type
+        })
       end
+
+      # def formatter_const
+      #   TwitterCldr::Formatters::TimeFormatter
+      # end
     end
 
   end

data/lib/twitter_cldr/localized/localized_timespan.rb

@@ -5,22 +5,67 @@
 
 module TwitterCldr
   module Localized
-
     class LocalizedTimespan < LocalizedObject
+
+      DEFAULT_TYPE = :default
+      APPROXIMATE_MULTIPLIER = 0.75
+
+      TIME_IN_SECONDS = {
+        :second => 1,
+        :minute => 60,
+        :hour   => 3600,
+        :day    => 86400,
+        :week   => 604800,
+        :month  => 2629743.83,
+        :year   => 31556926
+      }
+
       def initialize(seconds, options = {})
         super(seconds, options[:locale] || TwitterCldr.locale, options)
       end
 
       def to_s(options = {})
-        @formatter.format(@base_obj, options)
+        unit = options[:unit] || calculate_unit(base_obj.abs, options)
+        direction = options[:direction] || (base_obj < 0 ? :ago : :until)
+        type = options[:type] || DEFAULT_TYPE
+        number = calculate_time(base_obj, unit)
+
+        data_reader = TwitterCldr::DataReaders::TimespanDataReader.new(locale, number, {
+          :unit => unit,
+          :direction => direction,
+          :type => type
+        })
+
+        tokens = data_reader.tokenizer.tokenize(data_reader.pattern)
+        data_reader.formatter.format(tokens, number, options)
       end
 
       protected
 
-      def formatter_const
-        TwitterCldr::Formatters::TimespanFormatter
+      def calculate_unit(seconds, options = {})
+        approximate = options[:approximate]
+        approximate = false if approximate.nil?
+        multiplier = approximate ? APPROXIMATE_MULTIPLIER : 1
+
+        if seconds < (TIME_IN_SECONDS[:minute] * multiplier) then :second
+        elsif seconds < (TIME_IN_SECONDS[:hour] * multiplier) then :minute
+        elsif seconds < (TIME_IN_SECONDS[:day] * multiplier) then :hour
+        elsif seconds < (TIME_IN_SECONDS[:week] * multiplier) then :day
+        elsif seconds < (TIME_IN_SECONDS[:month] * multiplier) then :week
+        elsif seconds < (TIME_IN_SECONDS[:year] * multiplier) then :month
+        else :year end
       end
-    end
 
+      # 0 <-> 29 secs                                                   # => seconds
+      # 30 secs <-> 44 mins, 29 secs                                    # => minutes
+      # 44 mins, 30 secs <-> 23 hrs, 59 mins, 29 secs                   # => hours
+      # 23 hrs, 59 mins, 29 secs <-> 29 days, 23 hrs, 59 mins, 29 secs  # => days
+      # 29 days, 23 hrs, 59 mins, 29 secs <-> 1 yr minus 1 sec          # => months
+      # 1 yr <-> max time or date                                       # => years
+      def calculate_time(seconds, unit)
+        (seconds.to_f / TIME_IN_SECONDS[unit].to_f).abs.round.to_i
+      end
+
+    end
   end
 end

data/lib/twitter_cldr/normalization.rb

@@ -3,34 +3,23 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
+require 'eprun'
+
 module TwitterCldr
   module Normalization
-    autoload :Base,       'twitter_cldr/normalization/base'
-    autoload :Hangul,     'twitter_cldr/normalization/hangul'
-    autoload :QuickCheck, 'twitter_cldr/normalization/quick_check'
-    autoload :NFC,        'twitter_cldr/normalization/nfc'
-    autoload :NFD,        'twitter_cldr/normalization/nfd'
-    autoload :NFKC,       'twitter_cldr/normalization/nfkc'
-    autoload :NFKD,       'twitter_cldr/normalization/nfkd'
 
-    VALID_NORMALIZERS  = [:NFD, :NFKD, :NFC, :NFKC]
-    DEFAULT_NORMALIZER = :NFD
+    VALID_NORMALIZERS  = [:nfd, :nfkd, :nfc, :nfkc]
+    DEFAULT_NORMALIZER = :nfd
 
     class << self
 
       def normalize(string, options = {})
-        normalizer(options[:using] || DEFAULT_NORMALIZER).normalize(string)
-      end
-
-      private
-
-      def normalizer(normalizer_name)
-        const_name = normalizer_name.to_s.upcase.to_sym
+        form = options.fetch(:using, DEFAULT_NORMALIZER).to_s.downcase.to_sym
 
-        if VALID_NORMALIZERS.include?(const_name)
-          const_get(const_name)
+        if VALID_NORMALIZERS.include?(form)
+          Eprun.normalize(string, form)
         else
-          raise ArgumentError.new("#{normalizer_name.inspect} is not a valid normalizer (valid normalizers are #{VALID_NORMALIZERS.join(', ')})")
+          raise ArgumentError.new("#{form.inspect} is not a valid normalizer (valid normalizers are #{VALID_NORMALIZERS.join(', ')})")
         end
       end
 

data/lib/twitter_cldr/parsers/parser.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+
+    class UnexpectedTokenError < StandardError; end
+
+    # base class, not meant to be instantiated
+    class Parser
+
+      def parse(tokens, options = {})
+        @tokens = tokens
+        reset
+        do_parse(options)
+      end
+
+      def reset
+        @token_index = 0
+      end
+
+      private
+
+      def next_token(type)
+        unless current_token.type == type
+          raise UnexpectedTokenError.new("Unexpected token #{current_token.type} \"#{current_token.value}\"")
+        end
+
+        @token_index += 1
+
+        while current_token && empty?(current_token)
+          @token_index += 1
+        end
+
+        current_token
+      end
+
+      def empty?(token)
+        token.type == :plaintext && token.value == ""
+      end
+
+      def current_token
+        @tokens[@token_index]
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/parsers/segmentation_parser.rb

@@ -0,0 +1,137 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+
+    class SegmentationParser < Parser
+
+      RuleMatchData = Struct.new(:text, :boundary_offset)
+
+      class Rule
+        attr_accessor :string, :id
+      end
+
+      class BreakRule < Rule
+
+        attr_reader :left, :right
+
+        def initialize(left, right)
+          @left = left
+          @right = right
+        end
+
+        def match(str)
+          if left && left_match = left.match(str)
+            match_pos = left_match.offset(0).last
+
+            if right
+              if right_match = right.match(str[match_pos..-1])
+                RuleMatchData.new(
+                  left_match[0] + right_match[0],
+                  match_pos
+                )
+              end
+            else
+              RuleMatchData.new(str, str.size)
+            end
+          end
+        end
+
+        def boundary_symbol
+          :break
+        end
+
+      end
+
+      class NoBreakRule < Rule
+
+        attr_reader :regex
+
+        def initialize(regex)
+          @regex = regex
+        end
+
+        def match(str)
+          if match = regex.match(str)
+            RuleMatchData.new(match[0], match.offset(0).last)
+          end
+        end
+
+        def boundary_symbol
+          :no_break
+        end
+
+      end
+
+      private
+
+      def do_parse(options)
+        regex_token_lists = []
+        current_regex_tokens = []
+        boundary_symbol = nil
+
+        while current_token
+          case current_token.type
+            when :break, :no_break
+              boundary_symbol = current_token.type
+              regex_token_lists << current_regex_tokens
+              current_regex_tokens = []
+            else
+              current_regex_tokens << current_token
+          end
+
+          next_token(current_token.type)
+        end
+
+        regex_token_lists << current_regex_tokens
+
+        case boundary_symbol
+          when :break
+            BreakRule.new(
+              parse_regex(add_anchors(regex_token_lists[0]), options),
+              parse_regex(add_anchors(regex_token_lists[1]), options)
+            )
+          when :no_break
+            NoBreakRule.new(
+              parse_regex(add_anchors(regex_token_lists.flatten), options)
+            )
+        end
+      end
+
+      # only find matches from the beginning of the current string
+      def add_anchors(token_list)
+        token_list.insert(0, begin_token)
+      end
+
+      def begin_token
+        self.class.begin_token
+      end
+
+      def self.begin_token
+        @begin_token ||= TwitterCldr::Tokenizers::Token.new(
+          :type => :special_char, :value => "\\A"
+        )
+      end
+
+      def parse_regex(tokens, options)
+        unless tokens.empty?
+          TwitterCldr::Shared::UnicodeRegex.new(
+            regex_parser.parse(tokens, options)
+          )
+        end
+      end
+
+      def regex_parser
+        self.class.regex_parser
+      end
+
+      def self.regex_parser
+        @regex_parser ||= UnicodeRegexParser.new
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/parsers/symbol_table.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+
+    # This is really just a thin layer on top of Hash.
+    # Nice to have it abstracted in case we have to add custom behavior.
+    class SymbolTable
+
+      attr_reader :symbols
+
+      def initialize(symbols = {})
+        @symbols = symbols
+      end
+
+      def fetch(symbol)
+        symbols.fetch(symbol)
+      end
+
+      def add(symbol, value)
+        symbols[symbol] = value
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex/character_class.rb

@@ -0,0 +1,91 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+
+      # This is analogous to ICU's UnicodeSet class.
+      class CharacterClass < Component
+
+        GROUPING_PAIRS = {
+          :close_bracket => :open_bracket
+        }
+
+        # Character classes can include set operations (eg. union, intersection, etc).      
+        BinaryOperator = Struct.new(:operator, :left, :right)
+        UnaryOperator = Struct.new(:operator, :child)
+
+        class << self
+
+          def opening_types
+            @opening_types ||= GROUPING_PAIRS.values
+          end
+
+          def closing_types
+            @closing_types ||= GROUPING_PAIRS.keys
+          end
+
+          def opening_type_for(type)
+            GROUPING_PAIRS[type]
+          end
+
+        end
+
+        def initialize(root)
+          @root = root
+        end
+
+        def type
+          :character_class
+        end
+
+        def to_regexp_str
+          set_to_regex(to_set)
+        end
+
+        def to_set
+          evaluate(root)
+        end
+
+        private
+
+        attr_reader :root
+
+        def evaluate(node)
+          case node
+            when UnaryOperator, BinaryOperator
+              case node.operator
+                when :negate
+                  TwitterCldr::Shared::UnicodeRegex.valid_regexp_chars.subtract(
+                    evaluate(node.child)
+                  )
+                when :union, :pipe
+                  evaluate(node.left).union(
+                    evaluate(node.right)
+                  )
+                when :dash
+                  evaluate(node.left).difference(
+                    evaluate(node.right)
+                  )
+                when :ampersand
+                  evaluate(node.left).intersection(
+                    evaluate(node.right)
+                  )
+              end
+
+            else
+              if node
+                node.to_set
+              else
+                TwitterCldr::Utils::RangeSet.new([])
+              end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex/character_range.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+
+      # Regex character range eg. a-z or 0-9
+      # Can only exist in character classes
+      class CharacterRange < Component
+
+        attr_reader :initial, :final
+
+        def initialize(initial, final)
+          @initial = initial
+          @final = final
+        end
+
+        # Unfortunately, due to the ambiguity of having both character
+        # ranges and set operations in the same syntax (which both use
+        # the "-" operator and square brackets), we have to treat
+        # CharacterRange as both a token and an operand. This type method
+        # helps it behave like a token.
+        def type
+          :character_range
+        end
+
+        def to_set
+          TwitterCldr::Utils::RangeSet.new(
+            [initial.to_set.to_full_a.first..final.to_set.to_full_a.first]
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex/character_set.rb

@@ -0,0 +1,65 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+
+      # Can exist inside and outside of character classes
+      class CharacterSet < Component
+
+        attr_reader :property, :property_value
+
+        def initialize(text)
+          if (name_parts = text.split("=")).size == 2
+            @property = name_parts[0].downcase
+            @property_value = name_parts[1].to_sym
+          else
+            @property_value = text
+          end
+        end
+
+        def to_regexp_str
+          set_to_regex(to_set)
+        end
+
+        def to_set
+          codepoints.subtract(
+            TwitterCldr::Shared::UnicodeRegex.invalid_regexp_chars
+          )
+        end
+
+        private
+
+        def codepoints
+          if property
+            method = :"code_points_for_#{property}"
+
+            if TwitterCldr::Shared::CodePoint.respond_to?(method)
+              ranges = TwitterCldr::Shared::CodePoint.send(method, property_value)
+
+              if ranges
+                TwitterCldr::Utils::RangeSet.new(ranges)
+              else
+                raise UnicodeRegexParserError.new(
+                  "Couldn't find property '#{property}' containing property value '#{property_value}'"
+                )
+              end
+            else
+              raise UnicodeRegexParserError.new(
+                "Couldn't find property '#{property}"
+              )
+            end
+          else
+            TwitterCldr::Utils::RangeSet.new(
+              TwitterCldr::Shared::CodePoint.code_points_for_property_value(property_value)
+            )
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex/component.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+      class Component
+
+        protected
+
+        def to_utf8(codepoints)
+          # note: we do this for ruby 1.8 compatibility
+          # if we didn't have to support 1.8, we could do this instead:
+          # Array(codepoints).map { |cp| "\\u{#{cp.to_s(16).rjust(4, "0")}}"}.join
+          Array(codepoints).pack("U*").bytes.map { |s| "\\" + s.to_s(8) }.join
+        end
+
+        def range_to_regex(range)
+          if range.first.is_a?(Array)
+            array_to_regex(range)
+          else
+            "[#{to_utf8(range.first)}-#{to_utf8(range.last)}]"
+          end
+        end
+
+        def array_to_regex(arr)
+          arr.map { |elem| "(?:#{to_utf8(elem)})" }.join
+        end
+
+        def set_to_regex(set)
+          strs = set.to_a(true).uniq.map do |obj|
+            case obj
+              when Range
+                range_to_regex(obj)
+              when Array
+                array_to_regex(obj)
+              else
+                to_utf8(obj)
+            end
+          end
+
+          "(?:#{strs.join("|")})"
+        end
+
+      end
+    end
+  end
+end