twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/formatters/base.rb

@@ -1,47 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Formatters
-    class Base
-      attr_reader :tokenizer
-
-      def format(obj, options = {})
-        process_tokens(self.get_tokens(obj, options), obj, options)
-      end
-
-      protected
-
-      def process_tokens(tokens, obj, options = {})
-        result = ""
-
-        tokens.each_with_index do |token, index|
-          case token.type
-            when :composite
-              result << eval(process_tokens(token.tokens, obj)).to_s
-            when :pattern
-              result << self.result_for_token(token, index, obj, options)
-            else
-              if token.value.size > 0 && token.value[0].chr == "'" && token.value[-1].chr == "'"
-                result << token.value[1..-2]
-              else
-                result << token.value
-              end
-          end
-        end
-
-        result
-      end
-
-      def get_tokens(obj, options = {})
-        @tokenizer.tokens(options)
-      end
-
-      def extract_locale(options)
-        options[:locale] || TwitterCldr::DEFAULT_LOCALE
-      end
-    end
-  end
-end

data/lib/twitter_cldr/formatters/calendars/date_formatter.rb

@@ -1,19 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Formatters
-    class DateFormatter < DateTimeFormatter
-      def initialize(options = {})
-        locale = extract_locale(options)
-        cache_key = TwitterCldr::Utils.compute_cache_key("date", locale, options[:calendar_type])
-        @tokenizer = tokenizer_cache[cache_key] ||= TwitterCldr::Tokenizers::DateTokenizer.new(
-          :locale => locale,
-          :calendar_type => options[:calendar_type]
-        )
-      end
-    end
-  end
-end

data/lib/twitter_cldr/formatters/calendars/time_formatter.rb

@@ -1,19 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Formatters
-    class TimeFormatter < DateTimeFormatter
-      def initialize(options = {})
-        locale = extract_locale(options)
-        cache_key = TwitterCldr::Utils.compute_cache_key("time", locale, options[:calendar_type])
-        @tokenizer = tokenizer_cache[cache_key] ||= TwitterCldr::Tokenizers::TimeTokenizer.new(
-          :locale => locale,
-          :calendar_type => options[:calendar_type]
-        )
-      end
-    end
-  end
-end

data/lib/twitter_cldr/normalization/base.rb

@@ -1,37 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'hamster'
-
-module TwitterCldr
-  module Normalization
-    class Base
-
-      class << self
-
-        def normalize(string)
-          code_points = TwitterCldr::Utils::CodePoints.from_string(string)
-          normalized_code_points = normalize_code_points(code_points)
-          TwitterCldr::Utils::CodePoints.to_string(normalized_code_points)
-        end
-
-        def combining_class_for(code_point)
-          combining_class_cache[code_point] ||=
-            TwitterCldr::Shared::CodePoint.find(code_point).combining_class.to_i
-        rescue NoMethodError
-          0
-        end
-
-        protected
-
-        def combining_class_cache
-          @combining_class_cache ||= {}
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/normalization/hangul.rb

@@ -1,79 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Normalization
-    module Hangul
-
-      class << self
-
-        SBASE  = 0xAC00
-        LBASE  = 0x1100
-        VBASE  = 0x1161
-        TBASE  = 0x11A7
-
-        LCOUNT = 19
-        VCOUNT = 21
-        TCOUNT = 28
-
-        NCOUNT = VCOUNT * TCOUNT # 588
-        SCOUNT = LCOUNT * NCOUNT # 11172
-
-        LLIMIT = LBASE + LCOUNT  # 0x1113 = 4371
-        VLIMIT = VBASE + VCOUNT  # 0x1176 = 4470
-        TLIMIT = TBASE + TCOUNT  # 0x11C3 = 4547
-        SLIMIT = SBASE + SCOUNT  # 0xD7A4 = 55204
-
-        # Special composition for Hangul syllables. Documented in Section 3.12 at
-        # http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
-        #
-        def compose(code_points)
-          l = code_points.first - LBASE
-          v = code_points[1] - VBASE
-          t = code_points[2] ? code_points[2] - TBASE : 0  # T part may be missing, that's ok
-
-          SBASE + l * NCOUNT + v * TCOUNT + t
-        end
-
-        # Special decomposition for Hangul syllables. Documented in Section 3.12 at http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf
-        # Also see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm#Hangul_Implicit_CEs
-        #
-        def decompose(code_point)
-          decomposition_cache[code_point] ||= begin
-            l = code_point - SBASE
-
-            t = l % TCOUNT
-            l /= TCOUNT
-            v = l % VCOUNT
-            l /= VCOUNT
-
-            result = []
-
-            result << LBASE + l
-            result << VBASE + v
-            result << TBASE + t if t > 0
-
-            result
-          end
-        end
-
-        def hangul_syllable?(code_point)
-          (SBASE...SLIMIT).include?(code_point)
-        end
-
-        private
-
-        def syllable_cache
-          @syllable_cache ||= {}
-        end
-
-        def decomposition_cache
-          @decomposition_cache ||= {}
-        end
-
-      end
-    end
-  end
-end

data/lib/twitter_cldr/normalization/nfc.rb

@@ -1,24 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Normalization
-
-    # Implements normalization of a Unicode string to Normalization Form C (NFC).
-    # This normalization includes canonical decomposition followed by canonical composition.
-    #
-    class NFC < NFKC
-
-      class << self
-
-        def normalize_code_points(code_points)
-          compose(TwitterCldr::Normalization::NFD.normalize_code_points(code_points))
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/normalization/nfd.rb

@@ -1,26 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Normalization
-
-    # Implements normalization of a Unicode string to Normalization Form D (NFD).
-    # This normalization includes only canonical decomposition.
-    #
-    class NFD < NFKD
-
-      class << self
-
-        protected
-
-        def decompose?(unicode_data)
-          super && !unicode_data.compatibility_decomposition? # skip compatibility decompositions
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/normalization/nfkc.rb

@@ -1,114 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Normalization
-
-    # Implements normalization of a Unicode string to Normalization Form KC (NFKC).
-    # This normalization form includes compatibility decomposition followed by compatibility composition.
-    #
-    class NFKC < Base
-
-      class << self
-
-        VALID_HANGUL_SEQUENCES = [
-          [0, :lparts],
-          [1, :vparts],
-          [2, :tparts]
-        ]
-
-        def normalize_code_points(code_points)
-          compose(TwitterCldr::Normalization::NFKD.normalize_code_points(code_points))
-        end
-
-        protected
-
-        def compose(code_points)
-          final = []
-          hangul_code_points = []
-
-          code_points.each_with_index do |code_point, index|
-            final << code_point
-            hangul_type = TwitterCldr::Shared::CodePoint.hangul_type(code_point)
-            next_hangul_type = TwitterCldr::Shared::CodePoint.hangul_type(code_points[index + 1])
-
-            if valid_hangul_sequence?(hangul_code_points.size, hangul_type)
-              hangul_code_points << code_point
-              unless valid_hangul_sequence?(hangul_code_points.size, next_hangul_type)
-                next_hangul_type = nil
-              end
-            else
-              hangul_code_points.clear
-            end
-
-            if hangul_code_points.size > 1 && !next_hangul_type
-              final.pop(hangul_code_points.size)
-              final << compose_hangul(hangul_code_points)
-              hangul_code_points.clear
-            end
-          end
-
-          compose_normal(final)
-          final
-        end
-
-        def valid_hangul_sequence?(buffer_size, hangul_type)
-          VALID_HANGUL_SEQUENCES.include?([buffer_size, hangul_type])
-        end
-
-        def compose_hangul(code_points)
-          TwitterCldr::Normalization::Hangul.compose(code_points)
-        end
-
-        # Implements composition of Unicode code points following the guidelines here:
-        # http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf - Section 3.12
-        # Combining code points are combined with their base characters.  For example, "ñ"
-        # can be decomposed into 006E 0303, one code point for the "n" and the "˜" respectively.
-        # Composition reverses this process, turning 006E 0303 into a single 00F1 code point.
-        #
-        def compose_normal(code_points)
-          index = 1
-
-          while index < code_points.size
-            code_point = code_points[index]
-            combining_class = combining_class_for(code_point)
-            starter_index = find_starter_index(index, code_points)
-
-            # is this character blocked from combining with the last starter?
-            if starter_index < index - 1
-              previous_combining_class = combining_class_for(code_points[index - 1])
-              blocked = (previous_combining_class == 0) || (previous_combining_class >= combining_class)
-            else
-              blocked = false
-            end
-
-            unless blocked
-              # do a reverse-lookup for the decomposed code points
-              composite = TwitterCldr::Shared::CodePoint.for_canonical_decomposition([code_points[starter_index], code_point])
-
-              # check if two code points are canonically equivalent
-              if composite && !composite.excluded_from_composition?
-                # combine the characters
-                code_points[starter_index] = composite.code_point
-                code_points.delete_at(index)
-                index -= 1
-              end
-            end
-
-            index += 1
-          end
-        end
-
-        def find_starter_index(start_pos, code_points)
-          start_pos.times do |i|
-            return start_pos - i - 1 if combining_class_for(code_points[start_pos - i - 1]) == 0
-          end
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/normalization/nfkd.rb

@@ -1,120 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  # Normalization module includes algorithm for Unicode normalization. Basic information on this topic can be found in the
-  # Unicode Standard Annex #15 "Unicode Normalization Forms" at http://www.unicode.org/reports/tr15/. More detailed
-  # description is given in the section "3.11 Normalization Forms" of the Unicode Standard core specification. The
-  # latest version at the moment (for Unicode 6.1) is available at http://www.unicode.org/versions/Unicode6.1.0/ch03.pdf.
-  #
-  module Normalization
-
-    # Implements normalization of a Unicode string to Normalization Form KD (NFKD).
-    # This normalization form includes only compatibility decomposition.
-    #
-    class NFKD < Base
-
-      class << self
-
-        def normalize_code_points(code_points)
-          canonical_ordering(decompose(code_points))
-        end
-
-        protected
-
-        def decompose(code_points)
-          code_points.inject(Hamster.list) do |ret, code_point|
-            decompose_recursively(code_point).each do |decomp_cp|
-              ret = ret.cons(decomp_cp)
-            end
-            ret
-          end.reverse.to_a
-        end
-
-        # Recursively decomposes a given code point with the values in its Decomposition Mapping property.
-        #
-        def decompose_recursively(code_point)
-          unicode_data = TwitterCldr::Shared::CodePoint.find(code_point)
-          return [code_point] unless unicode_data
-
-          if unicode_data.hangul_type == :compositions
-            decompose_hangul(code_point)
-          else
-            decompose_regular(unicode_data)
-          end
-        end
-
-        # Decomposes regular (non-Hangul) code point.
-        #
-        def decompose_regular(unicode_data)
-          if decompose?(unicode_data)
-            unicode_data.decomposition.map { |code_point| decompose_recursively(code_point) }.flatten
-          else
-            [unicode_data.code_point]
-          end
-        end
-
-        def decompose?(unicode_data)
-          !!unicode_data.decomposition
-        end
-
-        def decompose_hangul(code_point)
-          TwitterCldr::Normalization::Hangul.decompose(code_point)
-        end
-
-        # Performs the Canonical Ordering Algorithm by stable sorting of every subsequence of combining code points
-        # (code points that have combining class greater than zero).
-        #
-        def canonical_ordering(code_points)
-          code_points_with_cc = code_points.map { |cp| [cp, combining_class_for(cp)] }
-
-          result = []
-          accum  = []
-
-          code_points_with_cc.each do |cp_with_cc|
-            if cp_with_cc[1] == 0
-              unless accum.empty?
-                result.concat(stable_sort(accum))
-                accum = []
-              end
-              result << cp_with_cc
-            else
-              accum << cp_with_cc
-            end
-          end
-
-          result.concat(stable_sort(accum)) unless accum.empty?
-          result.map { |cp_with_cc| cp_with_cc[0] }
-        end
-
-        # Performs stable sorting of a sequence of [code_point, combining_class] pairs. For sorting a regular bubble
-        # sort is used (with a small optimization that stops the algorithm if none of the elements were swapped during
-        # the iteration).
-        #
-        def stable_sort(code_points_with_cc)
-          n = code_points_with_cc.size - 2
-
-          code_points_with_cc.size.times do
-            swapped = false
-
-            (0..n).each do |j|
-              if code_points_with_cc[j][1] > code_points_with_cc[j + 1][1]
-                code_points_with_cc[j], code_points_with_cc[j + 1] = code_points_with_cc[j + 1], code_points_with_cc[j]
-                swapped = true
-              end
-            end
-
-            break unless swapped
-            n -= 1
-          end
-
-          code_points_with_cc
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/normalization/quick_check.rb

@@ -1,41 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Normalization
-
-    # This class isn't used anywhere because it was found that it negatively
-    # affects normalization performance.
-    module QuickCheck
-
-      class << self
-
-        def requires_normalization?(code_point, algorithm)
-          key = TwitterCldr::Utils.compute_cache_key(code_point, algorithm)
-          requires_cache[key] = if requires_cache[key].nil?
-            resource_for(algorithm).any? do |range|
-              range.include?(code_point)
-            end
-          else
-            requires_cache[key]
-          end
-        end
-
-        protected
-
-        def requires_cache
-          @requires_cache ||= {}
-        end
-
-        def resource_for(algorithm)
-          @resources ||= {}
-          @resources[algorithm] ||= TwitterCldr.get_resource("unicode_data", "#{algorithm.to_s.downcase}_quick_check")
-        end
-
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/tokenizers/base.rb

@@ -1,169 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Tokenizers
-    class Base
-      attr_reader :resource, :locale
-      attr_reader :token_splitter_regexes, :token_type_regexes, :paths
-      attr_accessor :type, :format, :placeholders
-
-      def initialize(options = {})
-        @locale = TwitterCldr.convert_locale(options[:locale] || TwitterCldr::DEFAULT_LOCALE)
-        init_resources
-        init_placeholders
-      end
-
-      protected
-
-      # Not to be confused with tokenize_pattern, which pulls out placeholders.  Tokenize_format actually splits a completely
-      # expanded format string into whatever parts are defined by the subclass's token type and token splitter regexes.
-      def tokenize_format(text)
-        text.split(token_splitter_regex_for(type)).each_with_index.inject([]) do |ret, (token, index)|
-          unless index == 0 && token == ""
-            regexes = token_type_regexes_for(type)
-
-            token_type = regexes.inject([]) do |match_ret, (token_type, matchers)|
-              match_ret << token_type if token =~ matchers[:regex]
-              match_ret
-            end.min { |a, b| regexes[a][:priority] <=> regexes[b][:priority] }
-
-            if token_type == :composite
-              content = token.match(regexes[token_type][:content])[1]
-              ret << CompositeToken.new(tokenize_format(content))
-            else
-              ret << Token.new(:value => token, :type => token_type)
-            end
-          end
-          ret
-        end
-      end
-
-      def token_type_regexes_for(type)
-        token_type_regexes[type] || token_type_regexes[:else]
-      end
-
-      def token_splitter_regex_for(type)
-        token_splitter_regexes[type] || token_splitter_regexes[:else]
-      end
-
-      def tokens_for(path, additional_cache_key_params = [])
-        tokens_for_pattern(pattern_for(traverse(path)), path, additional_cache_key_params)
-      end
-
-      def tokens_for_pattern(pattern, path, additional_cache_key_params = [])
-        cache_key = TwitterCldr::Utils.compute_cache_key(@locale, path.join('.'), type, format || "nil", *additional_cache_key_params)
-
-        unless token_cache.include?(cache_key)
-          result = []
-          tokens = expand_pattern(pattern)
-
-          tokens.each do |token|
-            if token.is_a?(Token) || token.is_a?(CompositeToken)
-              result << token
-            else
-              result += tokenize_format(token[:value])
-            end
-          end
-
-          token_cache[cache_key] = result
-        end
-
-        token_cache[cache_key]
-      end
-
-      def tokens_with_placeholders_for(key)
-        cache_key = TwitterCldr::Utils.compute_cache_key(@locale, key, type)
-
-        unless token_cache.include?(cache_key)
-          result = []
-          tokens = tokenize_pattern(pattern_for(traverse(key)))
-          tokens.each do |token|
-            result << token
-          end
-          token_cache[cache_key] = result
-        end
-
-        token_cache[cache_key]
-      end
-
-      def token_cache
-        @@token_cache ||= {}
-      end
-
-      def init_placeholders
-        @placeholders = {}
-      end
-
-      def traverse(path, hash = @resource)
-        TwitterCldr::Utils.traverse_hash(hash, path)
-      end
-
-      # expands all path symbols
-      def expand(current, haystack)
-        if current.is_a?(Symbol)
-          expand(traverse(current.to_s.split('.').map(&:to_sym), haystack), haystack)
-        elsif current.is_a?(Hash)
-          current.inject({}) do |ret, (key, val)|
-            ret[key] = expand(val, haystack)
-            ret
-          end
-        else
-          current
-        end
-      end
-
-      def expand_pattern(format_str)
-        if format_str.is_a?(Symbol)
-          # symbols mean another path was given
-          expand_pattern(pattern_for(traverse(format_str.to_s.split('.').map(&:to_sym))))
-        else
-          parts = tokenize_pattern(format_str)
-          final = []
-
-          parts.each do |part|
-            case part[:type]
-              when :placeholder
-                placeholder = choose_placeholder(part[:value], @placeholders)
-                final += placeholder ? placeholder.tokens(:type => type) : []
-              else
-                final << part
-            end
-          end
-
-          final
-        end
-      end
-
-      # Tokenize_pattern is supposed to take a pattern found in the YAML resource files and break it into placeholders and plaintext.
-      # Placeholders are delimited by single and double curly braces, plaintext is everything else.
-      def tokenize_pattern(pattern_str)
-        results = []
-        pattern_str.split(/(\{\{?\w*\}?\}|\'\w+\')/).each do |piece|
-          unless piece.empty?
-            case piece[0].chr
-              when "{"
-                results << { :value => piece, :type => :placeholder }
-              else
-                results << { :value => piece, :type => :plaintext }
-            end
-          end
-        end
-        results
-      end
-
-      def choose_placeholder(token, placeholders)
-        if token[0..1] == "{{"
-          token_value = token[2..-3]
-          found = placeholders.find { |placeholder| placeholder[:name].to_s == token_value }
-        else
-          found = placeholders[token[1..-2].to_i]
-        end
-
-        found ? found[:object] : nil
-      end
-    end
-  end
-end