twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/localized/localized_number.rb

@@ -29,9 +29,9 @@ module TwitterCldr
 
       def to_s(options = {})
         TwitterCldr::DataReaders::NumberDataReader.new(locale, {
-          :type => @type,
-          :format => @format
-        }).format_number(base_obj, options.merge(:type => @type))
+          type: @type,
+          format: @format
+        }).format_number(base_obj, options.merge(type: @type))
       end
 
       def plural_rule
@@ -46,8 +46,8 @@ module TwitterCldr
 
       def to_rbnf_s(group_name, rule_set_name)
         rbnf.format(base_obj, {
-          :rule_group => group_name,
-          :rule_set => rule_set_name
+          rule_group: group_name,
+          rule_set: rule_set_name
         })
       end
 
@@ -59,8 +59,8 @@ module TwitterCldr
 
       def to_type(target_type)
         self.class.new(base_obj, locale, {
-          :type => target_type,
-          :format => @format
+          type: target_type,
+          format: @format
         })
       end
 

data/lib/twitter_cldr/localized/localized_string.rb

@@ -36,6 +36,24 @@ module TwitterCldr
           localize(locale)
       end
 
+      def downcase
+        self.class.new(
+          TwitterCldr::Shared::Caser.downcase(@base_obj), locale
+        )
+      end
+
+      def upcase
+        self.class.new(
+          TwitterCldr::Shared::Caser.upcase(@base_obj), locale
+        )
+      end
+
+      def titlecase
+        self.class.new(
+          TwitterCldr::Shared::Caser.titlecase(@base_obj), locale
+        )
+      end
+
       def each_sentence
         if block_given?
           break_iterator.each_sentence(@base_obj) do |sentence|
@@ -46,6 +64,16 @@ module TwitterCldr
         end
       end
 
+      def each_word
+        if block_given?
+          break_iterator.each_word(@base_obj) do |word|
+            yield word.localize
+          end
+        else
+          to_enum(__method__)
+        end
+      end
+
       def code_points
         TwitterCldr::Utils::CodePoints.from_string(@base_obj)
       end
@@ -116,6 +144,10 @@ module TwitterCldr
         TwitterCldr::Shared::Territory.new(@base_obj)
       end
 
+      def script
+        TwitterCldr::Utils::ScriptDetector.detect_scripts(@base_obj).best_guess
+      end
+
       protected
 
       def escape_plural_interpolation(string)
@@ -129,7 +161,7 @@ module TwitterCldr
       end
 
       def break_iterator
-        @break_iterator ||= TwitterCldr::Shared::BreakIterator.new(locale)
+        @break_iterator ||= TwitterCldr::Segmentation::BreakIterator.new(locale)
       end
 
       def number_parser

data/lib/twitter_cldr/localized/localized_symbol.rb

@@ -11,6 +11,14 @@ module TwitterCldr
         TwitterCldr::Shared::Languages.from_code_for_locale(@base_obj, @locale)
       end
 
+      def as_locale
+        TwitterCldr::Shared::Locale.parse(@base_obj.to_s)
+      end
+
+      def as_territory
+        TwitterCldr::Shared::Territories::from_territory_code_for_locale(@base_obj, @locale)
+      end
+
       def is_rtl?
         TwitterCldr::Shared::Languages.is_rtl?(@base_obj)
       end
@@ -21,4 +29,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/localized/localized_time.rb

@@ -38,8 +38,8 @@ module TwitterCldr
       def data_reader_for(type, options = {})
         TwitterCldr::DataReaders::TimeDataReader.new(
           locale, options.merge({
-            :calendar_type => calendar_type,
-            :type => type
+            calendar_type: calendar_type,
+            type: type
           })
         )
       end

data/lib/twitter_cldr/localized/localized_timespan.rb

@@ -11,13 +11,13 @@ module TwitterCldr
       APPROXIMATE_MULTIPLIER = 0.75
 
       TIME_IN_SECONDS = {
-        :second => 1,
-        :minute => 60,
-        :hour   => 3600,
-        :day    => 86400,
-        :week   => 604800,
-        :month  => 2629743.83,
-        :year   => 31556926
+        second: 1,
+        minute: 60,
+        hour:   3600,
+        day:    86400,
+        week:   604800,
+        month:  2629743.83,
+        year:   31556926
       }
 
       def initialize(seconds, options = {})
@@ -31,9 +31,9 @@ module TwitterCldr
         number = calculate_time(base_obj, unit)
 
         data_reader = TwitterCldr::DataReaders::TimespanDataReader.new(locale, number, {
-          :unit => unit,
-          :direction => direction,
-          :type => type
+          unit: unit,
+          direction: direction,
+          type: type
         })
 
         tokens = data_reader.tokenizer.tokenize(data_reader.pattern)

data/lib/twitter_cldr/parsers/number_parser.rb

@@ -81,7 +81,7 @@ module TwitterCldr
       end
 
       def identify(text, group, decimal)
-        result = { :value => text }
+        result = { value: text }
         result[:type] = if self.class.is_numeric?(result[:value], "")
           :numeric
         else

data/lib/twitter_cldr/parsers/parser.rb

@@ -44,7 +44,11 @@ module TwitterCldr
       def current_token
         @tokens[@token_index]
       end
+
+      def eof?
+        @token_index >= @tokens.size
+      end
     end
 
   end
-end
+end

data/lib/twitter_cldr/parsers/unicode_regex/character_class.rb

@@ -11,10 +11,10 @@ module TwitterCldr
       class CharacterClass < Component
 
         GROUPING_PAIRS = {
-          :close_bracket => :open_bracket
+          close_bracket: :open_bracket
         }
 
-        # Character classes can include set operations (eg. union, intersection, etc).      
+        # Character classes can include set operations (eg. union, intersection, etc).
         BinaryOperator = Struct.new(:operator, :left, :right)
         UnaryOperator = Struct.new(:operator, :child)
 
@@ -50,10 +50,49 @@ module TwitterCldr
           evaluate(root)
         end
 
+        def codepoints
+          codepoints_from(root)
+        end
+
+        def to_s
+          stringify(root)
+        end
+
         private
 
         attr_reader :root
 
+        def codepoints_from(node)
+          case node
+            when UnaryOperator
+              codepoints_from(node.child)
+            when BinaryOperator
+              codepoints_from(node.left) + codepoints_from(node.right)
+            else
+              node.codepoints
+          end
+        end
+
+        def stringify(node)
+          case node
+            when UnaryOperator, BinaryOperator
+              op_str = case node.operator
+                when :negate then '^'
+                when :union, :pipe then ''
+                when :dash then '-'
+                when :ampersand then '&'
+              end
+
+              left = stringify(node.left)
+              right = stringify(node.right)
+
+              "#{left}#{op_str}#{right}"
+
+            else
+              node.to_s
+          end
+        end
+
         def evaluate(node)
           case node
             when UnaryOperator, BinaryOperator

data/lib/twitter_cldr/parsers/unicode_regex/character_range.rb

@@ -33,6 +33,14 @@ module TwitterCldr
           )
         end
 
+        def codepoints
+          to_set.to_full_a
+        end
+
+        def to_s
+          "#{initial.to_s}-#{final.to_s}"
+        end
+
       end
     end
   end

data/lib/twitter_cldr/parsers/unicode_regex/character_set.rb

@@ -10,12 +10,13 @@ module TwitterCldr
       # Can exist inside and outside of character classes
       class CharacterSet < Component
 
-        attr_reader :property, :property_value
+        include TwitterCldr::Shared
+
+        attr_reader :property_name, :property_value
 
         def initialize(text)
           if (name_parts = text.split("=")).size == 2
-            @property = name_parts[0].downcase
-            @property_value = name_parts[1].to_sym
+            @property_name, @property_value = name_parts
           else
             @property_value = text
           end
@@ -31,31 +32,73 @@ module TwitterCldr
           )
         end
 
+        def to_s
+          if property_value
+            "[:#{property_name}=#{property_value}:]"
+          else
+            "[:#{property_name}:]"
+          end
+        end
+
         private
 
         def codepoints
-          if property
-            method = :"code_points_for_#{property}"
-
-            if TwitterCldr::Shared::CodePoint.respond_to?(method)
-              ranges = TwitterCldr::Shared::CodePoint.send(method, property_value)
-
-              if ranges
-                TwitterCldr::Utils::RangeSet.new(ranges)
-              else
-                raise UnicodeRegexParserError.new(
-                  "Couldn't find property '#{property}' containing property value '#{property_value}'"
-                )
-              end
-            else
-              raise UnicodeRegexParserError.new(
-                "Couldn't find property '#{property}"
-              )
+          code_points = CodePoint.code_points_for_property(
+            *normalized_property
+          )
+
+          if code_points.empty?
+            raise UnicodeRegexParserError,
+              "Couldn't find property '#{property_name}' containing "\
+              "property value '#{property_value}'"
+          end
+
+          code_points
+        end
+
+        private
+
+        def normalized_property
+          property_value_candidates.each do |property_value|
+            prop_name, prop_value = normalized_property_name(
+              property_value, property_name_candidates
+            )
+
+            if prop_name
+              return [prop_name, prop_value]
             end
-          else
-            TwitterCldr::Utils::RangeSet.new(
-              TwitterCldr::Shared::CodePoint.code_points_for_property_value(property_value)
+          end
+
+          [nil, nil]
+        end
+
+        def normalized_property_name(property_value, property_name_candidates)
+          property_name_candidates.each do |property_name|
+            prop_name, prop_value = CodePoint.properties.normalize(
+              property_name, property_value
             )
+
+            if prop_name
+              return [prop_name, prop_value]
+            end
+          end
+
+          [nil, nil]
+        end
+
+        def property_name_candidates
+          if property_name
+            [property_name]
+          else
+            [property_value, 'General_Category', 'Script']
+          end
+        end
+
+        def property_value_candidates
+          if property_name && property_value
+            [property_value]
+          else
+            [property_value, nil].uniq
           end
         end
 

data/lib/twitter_cldr/parsers/unicode_regex/literal.rb

@@ -56,6 +56,10 @@ module TwitterCldr
           end
         end
 
+        def to_s
+          text
+        end
+
         private
 
         def set_for_special_char(char)

data/lib/twitter_cldr/parsers/unicode_regex/unicode_string.rb

@@ -31,11 +31,14 @@ module TwitterCldr
         end
 
         def to_regexp_str
-          cps = codepoints.is_a?(Array) ? codepoints : [codepoints]
-          array_to_regex(cps)
+          array_to_regex(Array(codepoints))
+        end
+
+        def to_s
+          to_regexp_str
         end
 
       end
     end
   end
-end
+end

data/lib/twitter_cldr/parsers/unicode_regex_parser.rb

@@ -28,11 +28,15 @@ module TwitterCldr
       private
 
       # Types that are allowed to be used in character ranges.
-      CHARACTER_CLASS_TOKEN_TYPES = [
+      RANGED_CHARACTER_CLASS_TOKEN_TYPES = [
         :variable, :character_set, :negated_character_set, :unicode_char,
         :multichar_string, :string, :escaped_character, :character_range
       ]
 
+      CHARACTER_CLASS_TOKEN_TYPES = RANGED_CHARACTER_CLASS_TOKEN_TYPES + [
+        :open_bracket, :special_char
+      ]
+
       NEGATED_TOKEN_TYPES = [
         :negated_character_set
       ]
@@ -47,27 +51,19 @@ module TwitterCldr
 
       def make_token(type, value = nil)
         TwitterCldr::Tokenizers::Token.new({
-          :type => type,
-          :value => value
+          type: type,
+          value: value
         })
       end
 
-      # Identifies regex ranges and makes implicit operators explicit
+      # Identifies regex ranges
       def preprocess(tokens)
         result = []
         i = 0
 
         while i < tokens.size
-          # Character class entities side-by-side are treated as unions. So
-          # are side-by-side character classes. Add a special placeholder token
-          # to help out the expression parser.
-          add_union = (valid_character_class_token?(result.last) && tokens[i].type != :close_bracket) ||
-            (result.last && result.last.type == :close_bracket && tokens[i].type == :open_bracket)
-
-          result << make_token(:union) if add_union
-
-          is_range = valid_character_class_token?(tokens[i]) &&
-            valid_character_class_token?(tokens[i + 2]) &&
+          is_range = valid_ranged_character_class_token?(tokens[i]) &&
+            valid_ranged_character_class_token?(tokens[i + 2]) &&
             tokens[i + 1].type == :dash
 
           if is_range
@@ -120,6 +116,10 @@ module TwitterCldr
         token && CHARACTER_CLASS_TOKEN_TYPES.include?(token.type)
       end
 
+      def valid_ranged_character_class_token?(token)
+        token && RANGED_CHARACTER_CLASS_TOKEN_TYPES.include?(token.type)
+      end
+
       def unary_operator?(token)
         token && UNARY_OPERATORS.include?(token.type)
       end
@@ -201,24 +201,9 @@ module TwitterCldr
         loop do
           case current_token.type
             when *CharacterClass.closing_types
-              last_operator = peek(operator_stack)
               open_count -= 1
-
-              until last_operator.type == CharacterClass.opening_type_for(current_token.type)
-                operator = operator_stack.pop
-
-                node = if unary_operator?(operator)
-                  unary_operator_node(operator.type, operand_stack.pop)
-                else
-                  binary_operator_node(
-                    operator.type, operand_stack.pop, operand_stack.pop
-                  )
-                end
-
-                operand_stack.push(node)
-                last_operator = peek(operator_stack)
-              end
-              operator_stack.pop
+              build_until_open(operator_stack, operand_stack)
+              add_implicit_union(operator_stack, open_count)
 
             when *CharacterClass.opening_types
               open_count += 1
@@ -228,6 +213,7 @@ module TwitterCldr
               operator_stack.push(current_token)
 
             else
+              add_implicit_union(operator_stack, open_count)
               operand_stack.push(
                 send(current_token.type, current_token)
               )
@@ -240,6 +226,53 @@ module TwitterCldr
         CharacterClass.new(operand_stack.pop)
       end
 
+      def build_until_open(operator_stack, operand_stack)
+        last_operator = peek(operator_stack)
+        opening_type = CharacterClass.opening_type_for(current_token.type)
+
+        until last_operator.type == opening_type
+          operator = operator_stack.pop
+          node = get_operator_node(operator, operand_stack)
+          operand_stack.push(node)
+          last_operator = peek(operator_stack)
+        end
+
+        operator_stack.pop
+      end
+
+      def get_operator_node(operator, operand_stack)
+        if operator.type == :dash && operand_stack.size < 2
+          get_non_range_dash_node(operator, operand_stack)
+        else
+          if unary_operator?(operator)
+            unary_operator_node(operator.type, operand_stack.pop)
+          else
+            binary_operator_node(
+              operator.type, operand_stack.pop, operand_stack.pop
+            )
+          end
+        end
+      end
+
+      # Most regular expression engines allow character classes
+      # to contain a literal hyphen caracter as the first character.
+      # For example, [-abc] is a legal expression. It denotes a
+      # character class that contains the letters '-', 'a', 'b',
+      # and 'c'. For example, /[-abc]*/.match('-ba') returns 0 in Ruby.
+      def get_non_range_dash_node(operator, operand_stack)
+        binary_operator_node(
+          :union, operand_stack.pop, string(make_token(:string, '-'))
+        )
+      end
+
+      def add_implicit_union(operator_stack, open_count)
+        if n = @tokens[@token_index + 1]
+          if valid_character_class_token?(n) && open_count > 0
+            operator_stack.push(make_token(:union))
+          end
+        end
+      end
+
       def peek(array)
         array.last
       end
@@ -259,4 +292,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/parsers.rb

@@ -9,6 +9,5 @@ module TwitterCldr
     autoload :SymbolTable,        'twitter_cldr/parsers/symbol_table'
     autoload :UnicodeRegexParser, 'twitter_cldr/parsers/unicode_regex_parser'
     autoload :NumberParser,       'twitter_cldr/parsers/number_parser'
-    autoload :SegmentationParser, 'twitter_cldr/parsers/segmentation_parser'
   end
-end
+end

data/lib/twitter_cldr/resources/custom_locales_resources_importer.rb

@@ -14,10 +14,10 @@ module TwitterCldr
       API_ENDPOINT = "http://translate.twitter.com/api/2/twitter/phrase/%s/translations.json"
 
       TIME_PERIODS = {
-        :day    => 19636,
-        :hour   => 19638,
-        :minute => 19634,
-        :second => 19639
+        day:    19636,
+        hour:   19638,
+        minute: 19634,
+        second: 19639
       }
 
       # Arguments:

data/lib/twitter_cldr/resources/download.rb

@@ -6,12 +6,9 @@
 module TwitterCldr
   module Resources
 
-    CLDR_URL = 'http://unicode.org/Public/cldr/26/core.zip'
-    ICU4J_URL = 'http://download.icu-project.org/files/icu4j/54.1/icu4j-54_1.jar'
-
-    # Use these instead to update collation and tailoring data
-    # CLDR_URL = 'http://unicode.org/Public/cldr/23.1/core.zip'
-    # ICU4J_URL = 'http://download.icu-project.org/files/icu4j/51.2/icu4j-51_2.jar'
+    CLDR_URL    = "http://unicode.org/Public/cldr/#{Versions.cldr_version}/core.zip"
+    ICU4J_URL   = "http://download.icu-project.org/files/icu4j/#{Versions.icu_version}/icu4j-#{Versions.icu_version.gsub('.', '_')}.jar"
+    UNICODE_URL = "ftp://ftp.unicode.org/Public/#{Versions.unicode_version}"
 
     class << self
 
@@ -51,6 +48,16 @@ module TwitterCldr
         path
       end
 
+      def download_unicode_data_if_necessary(path, url)
+        download_if_necessary(path, join_urls(UNICODE_URL, url))
+      end
+
+      private
+
+      def join_urls(*urls)
+        urls.map { |u| u.chomp('/') }.join('/')
+      end
+
     end
 
   end

data/lib/twitter_cldr/resources/language_codes_importer.rb

@@ -18,12 +18,12 @@ module TwitterCldr
       }
 
       KEYS_TO_STANDARDS = {
-        :iso_639_1      => :iso_639_1,
-        :iso_639_2      => :iso_639_2,
-        :iso_639_2_term => :iso_639_2,
-        :iso_639_3      => :iso_639_3,
-        :bcp_47         => :bcp_47,
-        :bcp_47_alt     => :bcp_47
+        iso_639_1:      :iso_639_1,
+        iso_639_2:      :iso_639_2,
+        iso_639_2_term: :iso_639_2,
+        iso_639_3:      :iso_639_3,
+        bcp_47:         :bcp_47,
+        bcp_47_alt:     :bcp_47
       }
 
       def initialize(input_path, output_path)
@@ -197,7 +197,7 @@ module TwitterCldr
         end
 
         language_codes_map.each do |name, codes|
-          table[:name][name] = { :name => name }.merge(codes)
+          table[:name][name] = { name: name }.merge(codes)
         end
 
         table[:name].values.each do |data|

data/lib/twitter_cldr/resources/loader.rb

@@ -6,6 +6,8 @@
 module TwitterCldr
   module Resources
 
+    class ResourceLoadError < StandardError; end
+
     class Loader
 
       def get_resource(*path)
@@ -94,7 +96,8 @@ module TwitterCldr
         if File.file?(file_path)
           File.read(file_path)
         else
-          raise ArgumentError.new("Resource '#{path}' not found.")
+          raise ResourceLoadError,
+            "Resource '#{path}' not found."
         end
       end