twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/shared/caser.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+    class Caser
+      REGEX = /./
+
+      class << self
+        def upcase(string)
+          string.gsub(REGEX, uppercasing_hash)
+        end
+
+        def downcase(string)
+          string.gsub(REGEX, lowercasing_hash)
+        end
+
+        # toTitlecase(X): Find the word boundaries in X according
+        # to Unicode Standard Annex #29, "Unicode Text Segmentation."
+        # For each word boundary, find the first cased character F
+        # following the word boundary. If F exists, map F to
+        # Titlecase_Mapping(F); then map all characters C between F
+        # and the following word boundary to Lowercase_Mapping(C).
+        def titlecase(string)
+          string.dup.tap do |result|
+            boundary_rule_set.each_boundary(result).each_cons(2) do |boundary_pair|
+              if cased_pos = first_cased(string, *boundary_pair)
+                result[cased_pos] = titlecasing_hash[result[cased_pos]]
+
+                (cased_pos + 1).upto(boundary_pair.last - 1) do |pos|
+                  result[pos] = lowercasing_hash[result[pos]]
+                end
+              end
+            end
+          end
+        end
+
+        private
+
+        def first_cased(string, start_pos, end_pos)
+          start_pos.upto(end_pos) do |pos|
+            return pos if cased?(string[pos])
+          end
+        end
+
+        def boundary_rule_set
+          @boundary_rule_set ||= Segmentation::RuleSet.load(:en, 'word')
+        end
+
+        def cased?(char)
+          props = CodePoint.properties_for_code_point(char.ord)
+          props.general_category.include?('Lt') ||
+            props.uppercase || props.lowercase
+        end
+
+        def uppercasing_hash
+          @uppercasing_hash ||= Hash.new do |hash, key|
+            memoize_value(:simple_uppercase_map, hash, key)
+          end
+        end
+
+        def lowercasing_hash
+          @lowercasing_hash ||= Hash.new do |hash, key|
+            memoize_value(:simple_lowercase_map, hash, key)
+          end
+        end
+
+        def titlecasing_hash
+          @titlecasing_hash ||= Hash.new do |hash, key|
+            memoize_value(:simple_titlecase_map, hash, key)
+          end
+        end
+
+        def memoize_value(field, hash, key)
+          cp = TwitterCldr::Shared::CodePoint.get(key.ord)
+          mapped_result = cp.send(field)
+          hash[key] = mapped_result ? mapped_result : key
+        end
+      end
+    end
+  end
+end

data/lib/twitter_cldr/shared/code_point.rb

@@ -6,87 +6,120 @@
 module TwitterCldr
   module Shared
 
-    CODE_POINT_FIELDS = [
-      :code_point,
-      :name,
-      :category,
-      :combining_class,
-      :bidi_class,
-      :decomposition,
-      :digit_value,
-      :non_decimal_digit_value,
-      :numeric_value,
-      :bidi_mirrored,
-      :unicode1_name,
-      :iso_comment,
-      :simple_uppercase_map,
-      :simple_lowercase_map,
-      :simple_titlecase_map
-    ]
-
     class CodePoint
-      DECOMPOSITION_DATA_INDEX = 5
       DECOMPOSITION_REGEX = /^(?:<(.+)>\s+)?(.+)?$/
+      MAX_CODE_POINT = 1_112_111
 
-      INDICES = [
-        :category, :bidi_class, :bidi_mirrored
-      ]
+      attr_reader :fields
 
-      PROPERTIES = [
-        :sentence_break, :line_break, :word_break
-      ]
+      def code_point
+        @fields[0]
+      end
 
-      attr_reader :fields
+      def name
+        @fields[1]
+      end
 
-      CODE_POINT_FIELDS.each_with_index do |field, idx|
-        unless field == :decomposition
-          define_method field do
-            fields[idx]
-          end
-        end
+      def category
+        @fields[2]
+      end
+
+      def combining_class
+        @fields[3]
+      end
+
+      def bidi_class
+        @fields[4]
       end
 
       def decomposition
         @decomposition ||= begin
-          decomp = fields[DECOMPOSITION_DATA_INDEX]
+          decomp = fields[5]
           if decomp =~ DECOMPOSITION_REGEX
             $2 && $2.split.map(&:hex)
           else
-            raise ArgumentError, "decomposition #{decomp.inspect} has invalid format"
+            raise ArgumentError,
+              "decomposition #{decomp.inspect} has invalid format"
           end
         end
       end
 
       def compatibility_decomposition_tag
         @compat_decomp_tag ||= begin
-          decomp = fields[DECOMPOSITION_DATA_INDEX]
+          decomp = fields[5]
           if decomp =~ DECOMPOSITION_REGEX
             $1
           else
-            raise ArgumentError, "decomposition #{decomp.inspect} has invalid format"
+            raise ArgumentError,
+              "decomposition #{decomp.inspect} has invalid format"
           end
         end
       end
 
+      def digit_value
+        @fields[6]
+      end
+
+      def non_decimal_digit_value
+        @fields[7]
+      end
+
+      def numeric_value
+        @fields[8]
+      end
+
+      def bidi_mirrored
+        @fields[9]
+      end
+
+      def unicode1_name
+        @fields[10]
+      end
+
+      def iso_comment
+        @fields[11]
+      end
+
+      def simple_uppercase_map
+        @uppercase ||= field_or_nil(12) do |val|
+          [val.to_i(16)].pack('U*')
+        end
+      end
+
+      def simple_lowercase_map
+        @lowercase ||= field_or_nil(13) do |val|
+          [val.to_i(16)].pack('U*')
+        end
+      end
+
+      def simple_titlecase_map
+        @titlecase ||= field_or_nil(14) do |val|
+          [val.to_i(16)].pack('U*')
+        end
+      end
+
       def initialize(fields)
         @fields = fields
       end
 
-      def compatibility_decomposition?
-        !!compatibility_decomposition_tag
+      def properties
+        self.class.properties.properties_for_code_point(code_point)
       end
 
-      def hangul_type
-        CodePoint.hangul_type(code_point)
-      end
+      private
 
-      def excluded_from_composition?
-        CodePoint.excluded_from_composition?(code_point)
+      def field_or_nil(index)
+        val = @fields[index]
+        if val && !val.empty?
+          yield val
+        end
       end
 
       class << self
 
-        def find(code_point)
+        include Enumerable
+
+        def get(code_point)
           code_point_cache[code_point] ||= begin
             target = get_block(code_point)
 
@@ -99,119 +132,46 @@ module TwitterCldr
           end
         end
 
-        # Methods that return a list of code points for the given property name.
-        INDICES.each do |index_name|
-          define_method :"code_points_for_#{index_name}" do |value|
-            get_index(index_name)[value]
-          end
+        def properties
+          @properties ||= TwitterCldr::Shared::PropertiesDatabase.new
         end
 
-        PROPERTIES.each do |property_name|
-          define_method :"code_points_for_#{property_name}" do |value|
-            get_property_data(property_name)[value]
-          end
-        end
-
-        # Search for code points wherein at least one property value contains prop_value.
-        # For example, if prop_value is set to :Zs, this method will return all code
-        # points that are considered spaces. If prop value is simply :Z, this method
-        # will return all code points who have a property value that contains :Z, i.e.
-        # spaces as well as line separators (:Zl) and paragraph separators (:Zp).
-        def code_points_for_property_value(prop_value)
-          index_key_cache[prop_value] ||= index_keys.inject([]) do |ret, (index_key, index_names)|
-            if index_key.to_s.include?(prop_value.to_s)
-              index_names.each do |index_name|
-                ret += get_index(index_name)[index_key]
-              end
-            end
-            ret
-          end
-        end
-
-        def for_canonical_decomposition(code_points)
-          if canonical_compositions.has_key?(code_points)
-            find(canonical_compositions[code_points])
-          end
+        def code_points_for_property(property_name, property_value = nil)
+          properties.code_points_for_property(
+            property_name, property_value
+          )
         end
 
-        def canonical_compositions
-          @canonical_compositions ||=
-            TwitterCldr.get_resource(:unicode_data, :canonical_compositions)
+        def properties_for_code_point(code_point)
+          properties.properties_for_code_point(code_point)
         end
 
-        def hangul_type(code_point)
-          hangul_type_cache[code_point] ||= begin
-            if code_point
-              [:lparts, :vparts, :tparts, :compositions].each do |type|
-                hangul_blocks[type].each do |range|
-                  return type if range.include?(code_point)
-                end
+        def each
+          if block_given?
+            (0..max).each do |cp|
+              if found_cp = get(cp)
+                yield found_cp
               end
-              nil
-            else
-              nil
             end
+          else
+            to_enum(__method__)
           end
         end
 
-        def excluded_from_composition?(code_point)
-          composition_exclusion_cache[code_point] ||=
-            composition_exclusions.any? { |exclusion| exclusion.include?(code_point) }
+        def max
+          MAX_CODE_POINT
         end
 
         private
 
-        def index_key_cache
-          @index_key_cache ||= {}
-        end
-
-        def index_keys
-          @index_keys ||= TwitterCldr.get_resource(:unicode_data, :indices, "keys")
-        end
-
-        def get_index(index_name)
-          index_cache[index_name] ||= TwitterCldr.get_resource(
-            :unicode_data, :indices, index_name
-          )
-        end
-
-        def get_property_data(property_name)
-          property_data_cache[property_name] ||= TwitterCldr.get_resource(
-            :unicode_data, :properties, property_name
-          )
-        end
-
-        def index_cache
-          @index_cache ||= {}
-        end
-
-        def property_data_cache
-          @property_data_cache ||= {}
-        end
-
-        def hangul_type_cache
-          @hangul_type_cache ||= {}
-        end
-
         def code_point_cache
           @code_point_cache ||= {}
         end
 
-        def composition_exclusion_cache
-          @composition_exclusion_cache ||= {}
-        end
-
-        def hangul_blocks
-          @hangul_blocks ||= TwitterCldr.get_resource(:unicode_data, :hangul_blocks)
-        end
-
-        def composition_exclusions
-          @composition_exclusions ||= TwitterCldr.get_resource(:unicode_data, :composition_exclusions)
-        end
-
         def get_block(code_point)
-          block_cache[code_point] ||=
-            blocks.detect { |_, range|  range.include?(code_point) }
+          block_cache[code_point] ||= blocks.detect do |_, range|
+            range.include?(code_point)
+          end
         end
 
         def block_cache
@@ -219,7 +179,9 @@ module TwitterCldr
         end
 
         def blocks
-          @blocks ||= TwitterCldr.get_resource(:unicode_data, :blocks)
+          @blocks ||= TwitterCldr.get_resource(
+            :unicode_data, :blocks
+          )
         end
 
         # Check if block constitutes a range. The code point beginning a range will have a name enclosed in <>, ending with 'First'
@@ -239,4 +201,4 @@ module TwitterCldr
       end
     end
   end
-end
+end

data/lib/twitter_cldr/shared/currencies.rb

@@ -18,11 +18,11 @@ module TwitterCldr
 
           if data
             result = {
-              :currency    => currency_code,
-              :name        => data[:one],
-              :cldr_symbol => data[:symbol] || currency_code.to_s,
-              :symbol      => data[:symbol] || currency_code.to_s,
-              :code_points => (data[:symbol] || currency_code.to_s).unpack("U*")
+              currency:    currency_code,
+              name:        data[:one],
+              cldr_symbol: data[:symbol] || currency_code.to_s,
+              symbol:      data[:symbol] || currency_code.to_s,
+              code_points: (data[:symbol] || currency_code.to_s).unpack("U*")
             }
 
             result.merge!(symbol_data) if symbol_data

data/lib/twitter_cldr/shared/language_codes.rb

@@ -33,11 +33,11 @@ module TwitterCldr
         end
 
         def from_language(language, standard)
-          convert(language, :from => NAME_STANDARD, :to => standard)
+          convert(language, from: NAME_STANDARD, to: standard)
         end
 
         def to_language(code, standard)
-          convert(code, :from => standard, :to => NAME_STANDARD).to_s
+          convert(code, from: standard, to: NAME_STANDARD).to_s
         end
 
         def standards_for(code, standard)

data/lib/twitter_cldr/shared/languages.rb

@@ -52,4 +52,4 @@ module TwitterCldr
 
     end
   end
-end
+end

data/lib/twitter_cldr/shared/likely_subtags.rb

@@ -0,0 +1,104 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+
+    class UnrecognizedSubtagsError < StandardError; end
+
+    class LikelySubtags
+      class << self
+        def locale_for(locale_text)
+          locale = Locale.parse(locale_text)
+          lookup(locale)
+        end
+
+        private
+
+        # Try each of the following in order (where the fields exist). The
+        # notation field³ means field¹ if it exists, otherwise field².
+        #
+        # 1. Lookup language¹ _ script¹ _ region¹. If in the table, return the
+        #    language² _ script² _ region² + variants.
+        #
+        # 2. Lookup language¹ _ script¹. If in the table, return language² _
+        #    script² _ region³ + variants.
+        #
+        # 3. Lookup language¹ _ region¹. If in the table, return language² _
+        #    script³ _ region² + variants.
+        #
+        # 4. Lookup language¹. If in the table, return language² _ script³ _
+        #    region³ + variants.
+        def lookup(locale)
+          first_lookup(locale) ||
+            second_lookup(locale) ||
+            third_lookup(locale) ||
+            fourth_lookup(locale) ||
+            raise(
+              UnrecognizedSubtagsError,
+              "couldn't find matching subtags for #{locale}"
+            )
+        end
+
+        def first_lookup(locale)
+          if locale.language && locale.script && locale.region
+            code = [locale.language, locale.script, locale.region].join('_')
+
+            if replacement = subtags_resource[code.to_sym]
+              language2, script2, region2 = Locale.split(replacement)
+              Locale.new(language2, script2, region2, locale.variants)
+            end
+          end
+        end
+
+        def second_lookup(locale)
+          if locale.language && locale.script
+            code = [locale.language, locale.script].join('_')
+
+            if replacement = subtags_resource[code.to_sym]
+              language2, script2, region2 = Locale.split(replacement)
+              Locale.new(
+                language2, script2, locale.region || region2, locale.variants
+              )
+            end
+          end
+        end
+
+        def third_lookup(locale)
+          if locale.language && locale.region
+            code = [locale.language, locale.region].join('_')
+
+            if replacement = subtags_resource[code.to_sym]
+              language2, script2, region2 = Locale.split(replacement)
+              Locale.new(
+                language2, locale.script || script2, region2, locale.variants
+              )
+            end
+          end
+        end
+
+        def fourth_lookup(locale)
+          if locale.language
+            if replacement = subtags_resource[locale.language.to_sym]
+              language2, script2, region2 = Locale.split(replacement)
+              Locale.new(
+                language2,
+                locale.script || script2,
+                locale.region || region2,
+                locale.variants
+              )
+            end
+          end
+        end
+
+        def subtags_resource
+          @subtags_resource ||=
+            TwitterCldr.get_resource('shared', 'likely_subtags')[:subtags]
+        end
+      end
+    end
+
+  end
+end