twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/parsers/unicode_regex/literal.rb

@@ -0,0 +1,83 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+      class Literal < Component
+
+        attr_reader :text
+
+        # ord is good enough (don't need unpack) because ASCII chars
+        # have the same numbers as their unicode equivalents
+        def self.ordinalize(char)
+          if char.respond_to?(:ord)
+            char.ord
+          else
+            char[0]
+          end
+        end
+
+        SPECIAL_CHARACTERS = {
+          "s" => [32],  # space
+          "t" => [9],   # tab
+          "r" => [13],  # carriage return
+          "n" => [10],  # newline
+          "f" => [12],  # form feed
+          "d" => ("0".."9").to_a.map { |c| ordinalize(c) },
+          "w" => (("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + ["_"]).map { |c| ordinalize(c) }
+        }
+
+        def initialize(text)
+          @text = text
+        end
+
+        def to_regexp_str
+          text
+        end
+
+        def to_set
+          if text =~ /^\\/
+            special_char = text[1..-1]
+
+            if SPECIAL_CHARACTERS.include?(special_char.downcase)
+              set_for_special_char(special_char)
+            else
+              TwitterCldr::Utils::RangeSet.from_array([
+                self.class.ordinalize(special_char)
+              ])
+            end
+          else
+            TwitterCldr::Utils::RangeSet.from_array([
+              self.class.ordinalize(text)
+            ])
+          end
+        end
+
+        private
+
+        def set_for_special_char(char)
+          special_char_set_cache[char] ||= begin
+            chars = TwitterCldr::Utils::RangeSet.from_array(
+              SPECIAL_CHARACTERS[char.downcase]
+            )
+
+            if char.upcase == char
+              UnicodeRegex.valid_regexp_chars.subtract(chars)
+            else
+              chars
+            end
+          end
+        end
+
+        def special_char_set_cache
+          @@special_char_set_cache ||= {}
+        end
+
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex/unicode_string.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+    class UnicodeRegexParser
+
+      # unicode_char, escaped_char, string, multichar_string
+      # Can exist inside and outside of character classes
+      class UnicodeString < Component
+
+        attr_reader :codepoints
+
+        def initialize(codepoints)
+          @codepoints = codepoints
+        end
+
+        def to_set
+          # If the number of codepoints is greater than 1, treat them as a
+          # group (eg. multichar string). This is definitely a hack in that
+          # it means there has to be special logic in RangeSet that deals
+          # with data types that aren't true integer ranges. I can't think
+          # of any other way to support multichar strings :(
+          if codepoints.size > 1
+            TwitterCldr::Utils::RangeSet.new([codepoints..codepoints])
+          else
+            TwitterCldr::Utils::RangeSet.new([codepoints.first..codepoints.first])
+          end
+        end
+
+        def to_regexp_str
+          cps = codepoints.is_a?(Array) ? codepoints : [codepoints]
+          array_to_regex(codepoints)
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/parsers/unicode_regex_parser.rb

@@ -0,0 +1,262 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Parsers
+
+    class UnicodeRegexParserError < StandardError; end
+
+    class UnicodeRegexParser < Parser
+
+      autoload :Component,      "twitter_cldr/parsers/unicode_regex/component"
+      autoload :CharacterClass, "twitter_cldr/parsers/unicode_regex/character_class"
+      autoload :CharacterRange, "twitter_cldr/parsers/unicode_regex/character_range"
+      autoload :CharacterSet,   "twitter_cldr/parsers/unicode_regex/character_set"
+      autoload :Literal,        "twitter_cldr/parsers/unicode_regex/literal"
+      autoload :UnicodeString,  "twitter_cldr/parsers/unicode_regex/unicode_string"
+
+      def parse(tokens, options = {})
+        super(
+          preprocess(
+            substitute_variables(tokens, options[:symbol_table])
+          ), options
+        )
+      end
+
+      private
+
+      # Types that are allowed to be used in character ranges.
+      CHARACTER_CLASS_TOKEN_TYPES = [
+        :variable, :character_set, :negated_character_set, :unicode_char,
+        :multichar_string, :string, :escaped_character, :character_range
+      ]
+
+      NEGATED_TOKEN_TYPES = [
+        :negated_character_set
+      ]
+
+      BINARY_OPERATORS = [
+        :pipe, :ampersand, :dash, :union
+      ]
+
+      UNARY_OPERATORS = [
+        :negate
+      ]
+
+      def make_token(type, value = nil)
+        TwitterCldr::Tokenizers::Token.new({
+          :type => type,
+          :value => value
+        })
+      end
+
+      # Identifies regex ranges and makes implicit operators explicit
+      def preprocess(tokens)
+        result = []
+        i = 0
+
+        while i < tokens.size
+          # Character class entities side-by-side are treated as unions. So
+          # are side-by-side character classes. Add a special placeholder token
+          # to help out the expression parser.
+          add_union = (valid_character_class_token?(result.last) && tokens[i].type != :close_bracket) ||
+            (result.last && result.last.type == :close_bracket && tokens[i].type == :open_bracket)
+
+          result << make_token(:union) if add_union
+
+          is_range = valid_character_class_token?(tokens[i]) &&
+            valid_character_class_token?(tokens[i + 2]) &&
+            tokens[i + 1].type == :dash
+
+          if is_range
+            initial = send(tokens[i].type, tokens[i])
+            final = send(tokens[i + 2].type, tokens[i + 2])
+            result << make_character_range(initial, final)
+            i += 3
+          else
+            if negated_token?(tokens[i])
+              result += [
+                make_token(:open_bracket),
+                make_token(:negate),
+                tokens[i],
+                make_token(:close_bracket)
+              ]
+            else
+              result << tokens[i]
+            end
+
+            i += 1
+          end
+        end
+
+        result
+      end
+
+      def substitute_variables(tokens, symbol_table)
+        return tokens unless symbol_table
+        tokens.inject([]) do |ret, token|
+          if token.type == :variable && sub = symbol_table.fetch(token.value)
+            # variables can themselves contain references to other variables
+            # note: this could be cached somehow
+            ret += substitute_variables(sub, symbol_table)
+          else
+            ret << token
+          end
+          ret
+        end
+      end
+
+      def make_character_range(initial, final)
+        CharacterRange.new(initial, final)
+      end
+
+      def negated_token?(token)
+        token && NEGATED_TOKEN_TYPES.include?(token.type)
+      end
+
+      def valid_character_class_token?(token)
+        token && CHARACTER_CLASS_TOKEN_TYPES.include?(token.type)
+      end
+
+      def unary_operator?(token)
+        token && UNARY_OPERATORS.include?(token.type)
+      end
+
+      def binary_operator?(token)
+        token && BINARY_OPERATORS.include?(token.type)
+      end
+
+      def do_parse(options)
+        elements = []
+
+        while current_token
+          case current_token.type
+            when :open_bracket
+              elements << character_class
+            when :union
+              next_token(:union)
+            else
+              elements << send(current_token.type, current_token)
+              next_token(current_token.type)
+          end
+        end
+
+        elements
+      end
+
+      def character_set(token)
+        CharacterSet.new(
+          token.value.gsub(/^\\p/, "").gsub(/[\{\}\[\]:]/, "")
+        )
+      end
+
+      def negated_character_set(token)
+        CharacterSet.new(
+          token.value.gsub(/^\\[pP]/, "").gsub(/[\{\}\[\]:^]/, "")
+        )
+      end
+
+      def unicode_char(token)
+        UnicodeString.new(
+          [token.value.gsub(/^\\u/, "").gsub(/[\{\}]/, "").to_i(16)]
+        )
+      end
+
+      def string(token)
+        UnicodeString.new(
+          token.value.unpack("U*")
+        )
+      end
+
+      def multichar_string(token)
+        UnicodeString.new(
+          token.value.gsub(/[\{\}]/, "").unpack("U*")
+        )
+      end
+
+      def escaped_character(token)
+        Literal.new(token.value)
+      end
+
+      def special_char(token)
+        Literal.new(token.value)
+      end
+
+      alias :negate :special_char
+      alias :pipe :special_char
+      alias :ampersand :special_char
+
+      # current_token is already a CharacterRange object
+      def character_range(token)
+        token
+      end
+
+      def character_class
+        operator_stack = []
+        operand_stack = []
+        open_count = 0
+
+        loop do
+          case current_token.type
+            when *CharacterClass.closing_types
+              last_operator = peek(operator_stack)
+              open_count -= 1
+
+              until last_operator.type == CharacterClass.opening_type_for(current_token.type)
+                operator = operator_stack.pop
+
+                node = if unary_operator?(operator)
+                  unary_operator_node(operator.type, operand_stack.pop)
+                else
+                  binary_operator_node(
+                    operator.type, operand_stack.pop, operand_stack.pop
+                  )
+                end
+
+                operand_stack.push(node)
+                last_operator = peek(operator_stack)
+              end
+              operator_stack.pop
+
+            when *CharacterClass.opening_types
+              open_count += 1
+              operator_stack.push(current_token)
+
+            when *(BINARY_OPERATORS + UNARY_OPERATORS)
+              operator_stack.push(current_token)
+
+            else
+              operand_stack.push(
+                send(current_token.type, current_token)
+              )
+          end
+
+          next_token(current_token.type)
+          break if operator_stack.empty? && open_count == 0
+        end
+
+        CharacterClass.new(operand_stack.pop)
+      end
+
+      def peek(array)
+        array.last
+      end
+
+      def binary_operator_node(operator, right, left)
+        CharacterClass::BinaryOperator.new(
+          operator, left, right
+        )
+      end
+
+      def unary_operator_node(operator, child)
+        CharacterClass::UnaryOperator.new(
+          operator, child
+        )
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/parsers.rb

@@ -5,6 +5,10 @@
 
 module TwitterCldr
   module Parsers
-    autoload :NumberParser, 'twitter_cldr/parsers/number_parser'
+    autoload :Parser,             'twitter_cldr/parsers/parser'
+    autoload :SymbolTable,        'twitter_cldr/parsers/symbol_table'
+    autoload :UnicodeRegexParser, 'twitter_cldr/parsers/unicode_regex_parser'
+    autoload :NumberParser,       'twitter_cldr/parsers/number_parser'
+    autoload :SegmentationParser, 'twitter_cldr/parsers/segmentation_parser'
   end
 end

data/lib/twitter_cldr/resources/casefolder.rb.erb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+    class Casefolder
+      class << self
+
+        CASEFOLDING_REGEX_C = /<%= casefolding_char_class_for("C") %>/
+        CASEFOLDING_REGEX_F = /<%= casefolding_char_class_for("F") %>/
+        CASEFOLDING_REGEX_S = /<%= casefolding_char_class_for("S") %>/
+        CASEFOLDING_REGEX_T = /<%= casefolding_char_class_for("T") %>/
+
+        CASEFOLDING_HASH = <%= inspect_hash_in_lines(casefolding_hash_for(["C", "F", "S"]), 8, 5) %>
+        CASEFOLDING_HASH_T = CASEFOLDING_HASH.merge(
+          <%= inspect_hash_in_lines(casefolding_hash_for(["T"]), 9, 6) %>
+        )
+
+        def simple_casefold(str, t = false)
+          perform_casefold(str, simple_casefold_regex, t)
+        end
+
+        def full_casefold(str, t = false)
+          perform_casefold(str, full_casefold_regex, t)
+        end
+
+        alias :casefold :full_casefold
+
+        def common_casefold(str)
+          perform_casefold(str, CASEFOLDING_REGEX_C, false)
+        end
+
+        private
+
+        def perform_casefold(str, regex, t)
+          regex = regex_with_t(regex) if t
+          casefolding_hash = t ? CASEFOLDING_HASH_T : CASEFOLDING_HASH
+
+          str.gsub(regex) do |s|
+            s.unpack("U*").inject([]) do |ret, ss|
+              ret + casefolding_hash[ss]
+            end.pack("U*")
+          end
+        end
+
+        def simple_casefold_regex
+          @simple_casefold_regex ||= Regexp.union(CASEFOLDING_REGEX_C, CASEFOLDING_REGEX_S)
+        end
+
+        def full_casefold_regex
+          @full_casefold_regex ||= Regexp.union(CASEFOLDING_REGEX_C, CASEFOLDING_REGEX_F)
+        end
+
+        def regex_with_t(regex)
+          @regex_with_t_cache[regex.source] ||=
+            Regexp.union(regex, CASEFOLDING_REGEX_T)
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/resources/casefolder_class_generator.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'erb'
+
+module TwitterCldr
+  module Resources
+    class CasefolderClassGenerator
+
+      attr_reader :template_file, :output_dir
+
+      def initialize(template_file, output_dir)
+        @template_file = template_file
+        @output_dir = output_dir
+      end
+
+      def generate
+        output_file = File.basename(template_file).chomp(".erb")
+        File.open(File.join(output_dir, output_file), "w+") do |f|
+          f.write(
+            ERB.new(File.read(template_file)).result(binding)
+          )
+        end
+      end
+
+      def casefolding_char_class_for(status)
+        to_regex_char_sequence(casefolding_data_for(status))
+      end
+
+      def casefolding_hash_for(statuses)
+        statuses.inject({}) do |ret, status|
+          ret.merge!(casefolding_data_for(status))
+          ret
+        end
+      end
+
+      def inspect_hash_in_lines(hash, per_line, indent)
+        str = "{\n#{"  " * indent}"
+        hash.each_with_index do |(key, val), idx|
+          if idx > 0 && idx % per_line == 0
+            str << "\n#{"  " * indent}"
+          end
+          str << "#{key.inspect}=>#{val.inspect}"
+          str << ", " if idx != (hash.size - 1)
+        end
+        str << "\n#{"  " * (indent - 1)}}"
+      end
+
+      private
+
+      def to_regex_char_sequence(casefold_data)
+        casefold_data.map { |(source, _)| to_utf8(source) }.join("|")
+      end
+
+      def to_utf8(obj)
+        arr = obj.is_a?(Array) ? obj : [obj]
+        arr.pack("U*").bytes.to_a.map { |s| "\\" + s.to_s(8) }.join
+      end
+
+      def casefolding_data_for(status)
+        resource.inject({}) do |ret, data|
+          ret[data[:source]] = data[:target] if data[:status] == status
+          ret
+        end
+      end
+
+      def resource
+        @@resource ||= TwitterCldr.get_resource("unicode_data", "casefolding")
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/download.rb

@@ -3,13 +3,11 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'cldr/download'
-
 module TwitterCldr
   module Resources
 
-    CLDR_URL = 'http://unicode.org/Public/cldr/23.1/core.zip'
-    ICU4J_URL = 'http://download.icu-project.org/files/icu4j/51.2/icu4j-51_2.jar'
+    CLDR_URL = 'http://unicode.org/Public/cldr/24/core.zip'
+    ICU4J_URL = 'http://download.icu-project.org/files/icu4j/52.1/icu4j-52_1.jar'
 
     class << self
 
@@ -29,6 +27,14 @@ module TwitterCldr
         if File.directory?(path)
           puts "Using CLDR data from '#{path}'."
         else
+          begin
+            require 'zip'
+          rescue LoadError
+            raise StandardError.new("Unable to require 'zip'. Please switch to at least Ruby 1.9, then rebundle and try again.")
+          end
+
+          require 'cldr/download'
+
           puts "Downloading CLDR data from '#{url}' to '#{path}'."
           Cldr.download(url, path)
         end

data/lib/twitter_cldr/resources/icu_based_importer.rb

@@ -0,0 +1,18 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'java'
+require 'twitter_cldr/resources/download'
+
+class IcuBasedImporter
+
+  protected
+
+  def require_icu4j(icu4j_path)
+    TwitterCldr::Resources.download_icu4j_if_necessary(icu4j_path)
+    require icu4j_path
+  end
+
+end

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -12,7 +12,9 @@ module TwitterCldr
 
     class LocalesResourcesImporter
 
-      COMPONENTS = %w[calendars languages numbers units plurals lists layout currencies territories]
+      # NOTE: units.yml was NOT updated to cldr 24 (too many significant changes) - add back in when appropriate
+      LOCALE_COMPONENTS = %w[calendars languages numbers plurals lists layout currencies territories rbnf]  # units
+      SHARED_COMPONENTS = %w[currency_digits_and_rounding rbnf_root numbering_systems segments_root]
 
       # Arguments:
       #
@@ -48,7 +50,7 @@ module TwitterCldr
       def import_components
         export_args = {
           :locales => TwitterCldr.supported_locales,
-          :components => COMPONENTS,
+          :components => LOCALE_COMPONENTS,
           :target => File.join(@output_path, 'locales'),
           :merge => true  # fill in the gaps, eg fill in sub-locales like en_GB with en
         }
@@ -61,7 +63,7 @@ module TwitterCldr
         end
 
         export_args = {
-          :components => ["currency_digits_and_rounding"],
+          :components => SHARED_COMPONENTS,
           :target => File.join(@output_path, 'shared'),
           :merge => true
         }
@@ -78,7 +80,16 @@ module TwitterCldr
         data = YAML.load(File.read(path))
 
         File.open(path, 'w:utf-8') do |output|
-          output.write(YAML.dump(TwitterCldr::Utils.deep_symbolize_keys(data)))
+          output.write(
+            # Quote all strings for compat with 1.8. This is important because
+            # RBNF syntax includes characters that are significant in the Yaml
+            # syntax, like >, <, etc. Psych doesn't have problems parsing them,
+            # but Syck does (ruby 1.8).
+            TwitterCldr::Utils::YAML.dump(TwitterCldr::Utils.deep_symbolize_keys(data), {
+              :quote_all_strings => true,
+              :use_natural_symbols => true
+            })
+          )
         end
       end
 
@@ -93,7 +104,7 @@ module TwitterCldr
 
         data = YAML.load(File.read(path))
         data.keys.each do |l|
-          data[l] = TwitterCldr::Utils::Territories.deep_normalize_territory_code_keys(data[l])
+          data[l] = TwitterCldr::Shared::Territories.deep_normalize_territory_code_keys(data[l])
         end
 
         File.open(path, 'w:utf-8') do |output|
@@ -125,15 +136,15 @@ module TwitterCldr
       end
 
       BUDDHIST_CALENDAR = {
-          'formats' => {
-              'date' => {
-                  'default' => :'calendars.buddhist.formats.date.medium',
-                  'full'    => { 'pattern' => 'EEEEที่ d MMMM G y' },
-                  'long'    => { 'pattern' => 'd MMMM พ.ศ. #{y + 543}' },
-                  'medium'  => { 'pattern' => 'd MMM #{y + 543}' },
-                  'short'   => { 'pattern' => 'd/M/#{y + 543}' }
-              }
+        'formats' => {
+          'date' => {
+            'default' => :'calendars.buddhist.formats.date.medium',
+            'full'    => { 'pattern' => 'EEEEที่ d MMMM G y' },
+            'long'    => { 'pattern' => 'd MMMM พ.ศ. #{y + 543}' },
+            'medium'  => { 'pattern' => 'd MMM #{y + 543}' },
+            'short'   => { 'pattern' => 'd/M/#{y + 543}' }
           }
+        }
       }
 
     end

data/lib/twitter_cldr/resources/normalization_quick_check_importer.rb

@@ -25,26 +25,13 @@ module TwitterCldr
       def import
         parse_props_file.each_pair do |algorithm, code_point_list|
           File.open(File.join(@output_path, "#{algorithm.downcase}_quick_check.yml"), "w+") do |f|
-            f.write(YAML.dump(rangify(partition_prop_list(code_point_list))))
+            f.write(YAML.dump(TwitterCldr::Utils::RangeSet.rangify(code_point_list)))
           end
         end
       end
 
       private
 
-      def rangify(lists)
-        lists.map { |list| (list.first..list.last) }
-      end
-
-      def partition_prop_list(list)
-        last_item = 0
-        list.inject([]) do |ret, item|
-          (item - last_item == 1) ? ret[-1] << item : ret << [item]
-          last_item = item
-          ret
-        end
-      end
-
       def parse_props_file
         check_table = {}
         cur_type = nil