twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/shared/property_normalizer.rb

@@ -0,0 +1,108 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+
+    class PropertyNormalizer
+      attr_reader :database
+
+      def initialize(database)
+        @database = database
+      end
+
+      def normalize(property_name, property_value = nil)
+        candidates = find_property_name_candidates(property_name)
+
+        if property_value
+          name, value = resolve_candidates(candidates, property_value)
+        else
+          candidates.each do |c|
+            if name = resolve_property_name_case(c)
+              break
+            end
+          end
+
+          value = nil
+        end
+
+        [name, value]
+      end
+
+      private
+
+      def resolve_candidates(property_name_candidates, property_value)
+        value_candidates = find_property_value_candidates(
+          property_name_candidates, property_value
+        )
+
+        resolve_name_value_candidates(
+          property_name_candidates, value_candidates
+        )
+      end
+
+      def resolve_name_value_candidates(property_name_candidates, value_candidates)
+        property_name_candidates.each do |name_candidate|
+          value_candidates.each do |value_candidate|
+            if cased_property_name = resolve_property_name_case(name_candidate)
+              if value_candidate
+                cased_property_value = resolve_property_value_case(
+                  cased_property_name, value_candidate
+                )
+
+                if cased_property_value
+                  return [
+                    cased_property_name, cased_property_value
+                  ]
+                end
+              else
+                if database.include?(cased_property_name, value_candidate)
+                  return [
+                    cased_property_name, value_candidate
+                  ]
+                end
+              end
+            end
+          end
+        end
+
+        []
+      end
+
+      def resolve_property_name_case(property_name)
+        name_idx = database.property_names.index do |name|
+          name.casecmp(property_name).zero?
+        end
+
+        database.property_names[name_idx] if name_idx
+      end
+
+      def resolve_property_value_case(property_name, property_value)
+        property_values = database.property_values_for(property_name)
+
+        if property_values
+          value_idx = property_values.index do |value|
+            value.casecmp(property_value).zero?
+          end
+
+          property_values[value_idx] if value_idx
+        end
+      end
+
+      def find_property_name_candidates(property_name)
+        aliases = PropertyNameAliases.aliases_for(property_name)
+        aliases << property_name
+        aliases.uniq
+      end
+
+      def find_property_value_candidates(property_name_candidates, property_value)
+        property_name_candidates.flat_map do |property_name|
+          PropertyValueAliases.aliases_for(property_name, property_value) + [property_value]
+        end.uniq
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/shared/property_set.rb

@@ -0,0 +1,113 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'forwardable'
+
+module TwitterCldr
+  module Shared
+    class PropertySet
+      include TwitterCldr::Shared::Properties
+      extend SingleForwardable
+
+      attr_reader :properties_hash
+
+      def_delegators :properties_hash, :empty?, :include?, :size
+
+      def initialize(properties_hash)
+        @properties_hash = properties_hash
+      end
+
+      def age
+        properties_hash.fetch('Age', ['Unassigned'])
+      end
+
+      def joining_type
+        properties_hash['Joining_Type'] ||= if general_category.empty?
+          [ArabicShaping.joining_type_for_general_category('xx')]
+        else
+          general_category.map do |gc|
+            ArabicShaping.joining_type_for_general_category(gc)
+          end
+        end
+      end
+
+      def bidi_paired_bracket_type
+        properties_hash['Bidi_Paired_Bracket_Type'] ||= [
+          BidiBrackets.bracket_types['N']
+        ]
+      end
+
+      def block
+        properties_hash['Block'] ||= ['No_Block']
+      end
+
+      def east_asian_width
+        properties_hash['East_Asian_Width'] ||= ['N']
+      end
+
+      def grapheme_cluster_break
+        properties_hash['Grapheme_Cluster_Break'] ||= ['Other']
+      end
+
+      def hangul_syllable_type
+        properties_hash['Hangul_Syllable_Type'] ||= ['Not_Applicable']
+      end
+
+      def indic_positional_category
+        properties_hash['Indic_Positional_Category'] ||= ['NA']
+      end
+
+      def indic_syllabic_category
+        properties_hash['Indic_Syllabic_Category'] ||= ['Other']
+      end
+
+      def jamo_short_name
+        properties_hash['Jamo_Short_Name'] ||= ['<none>']
+      end
+
+      def line_break
+        properties_hash['Line_Break'] ||= ['XX']
+      end
+
+      def general_category
+        properties_hash.fetch('General_Category', [])
+      end
+
+      def script_extensions
+        properties_hash['Script_Extensions'] ||= ['<script>']
+      end
+
+      def script
+        properties_hash['Script'] ||= ['Unknown']
+      end
+
+      def sentence_break
+        properties_hash['Sentence_Break'] ||= ['Other']
+      end
+
+      def word_break
+        properties_hash['Word_Break'] ||= ['Other']
+      end
+
+      module AdditionalPropertyMethods
+        CodePoint.properties.property_names.each do |property_name|
+          method_name = property_name.downcase
+
+          unless PropertySet.method_defined?(method_name)
+            define_method(method_name) do
+              if properties_hash.include?(property_name)
+                properties_hash[property_name] || true
+              else
+                false
+              end
+            end
+          end
+        end
+      end
+
+      include AdditionalPropertyMethods
+    end
+  end
+end

data/lib/twitter_cldr/shared/property_value_aliases.rb

@@ -0,0 +1,99 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+
+    class PropertyValueAliases
+      class << self
+
+        def abbreviated_alias_for(property_name, property_value)
+          (find_index_for(property_name, property_value) || {})
+            .fetch(property_value, {})
+            .fetch(:abbreviated_name, nil)
+        end
+
+        def long_alias_for(property_name, property_value)
+          (find_index_for(property_name, property_value) || {})
+            .fetch(property_value, {})
+            .fetch(:long_name, nil)
+        end
+
+        def numeric_alias_for(property_name, property_value)
+          (find_index_for(property_name, property_value) || {})
+            .fetch(property_value, {})
+            .fetch(:numeric, nil)
+        end
+
+        def aliases_for(property_name, property_value)
+          aliases = [
+            abbreviated_alias_for(property_name, property_value),
+            long_alias_for(property_name, property_value),
+            numeric_alias_for(property_name, property_value)
+          ].compact
+
+          aliases.delete(property_value)
+          aliases
+        end
+
+        private
+
+        def find_index_for(property_name, property_value)
+          indices_for(property_name).find do |index|
+            index.include?(property_value)
+          end
+        end
+
+        def indices_for(property_name)
+          [
+            abbreviated_index_for(property_name),
+            long_index_for(property_name),
+            numeric_index_for(property_name)
+          ].compact
+        end
+
+        def abbreviated_index_for(property_name)
+          abbreviated_indices[property_name] ||=
+            create_index(property_name, :abbreviated_name)
+        end
+
+        def long_index_for(property_name)
+          long_indices[property_name] ||=
+            create_index(property_name, :long_name)
+        end
+
+        def numeric_index_for(property_name)
+          numeric_indices[property_name] ||=
+            create_index(property_name, :numeric)
+        end
+
+        def create_index(property_name, field)
+          (resource[property_name] || {}).each_with_object({}) do |fields, ret|
+            ret[fields[field]] = fields
+          end
+        end
+
+        def abbreviated_indices
+          @abbreviated_indices ||= {}
+        end
+
+        def long_indices
+          @long_indices ||= {}
+        end
+
+        def numeric_indices
+          @numeric_indices ||= {}
+        end
+
+        def resource
+          @resource ||=
+            TwitterCldr.get_resource('unicode_data', 'property_value_aliases')
+        end
+
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/shared/unicode_regex.rb

@@ -12,7 +12,7 @@ module TwitterCldr
         def compile(str, modifiers = "", symbol_table = nil)
           new(
             parser.parse(tokenizer.tokenize(str), {
-              :symbol_table => symbol_table
+              symbol_table: symbol_table
             }), modifiers
           )
         end
@@ -57,17 +57,43 @@ module TwitterCldr
 
       def initialize(elements, modifiers = nil)
         @elements = elements
-        @modifiers = nil
+        @modifiers = modifiers
       end
 
       def to_regexp
-        @regexp ||= Regexp.new(to_regexp_str, modifiers)
+        @regexp ||= Regexp.new(to_regexp_str, modifier_union)
       end
 
       def to_regexp_str
         @regexp_str ||= elements.map(&:to_regexp_str).join
       end
 
+      def to_s
+        @elements.inject('') do |ret, element|
+          ret + element.to_s
+        end
+      end
+
+      private
+
+      def modifier_union
+        @modifier_union ||=
+          (modifiers || '').each_char.inject(0) do |ret, modifier_char|
+            ret | case modifier_char
+              when 'm'
+                Regexp::MULTILINE
+              when 'i'
+                Regexp::IGNORECASE
+              when 'x'
+                Regexp::EXTENDED
+              when 'n'
+                Regexp::NOENCODING
+              else
+                0
+            end
+          end
+      end
+
     end
   end
 end

data/lib/twitter_cldr/shared.rb

@@ -6,18 +6,26 @@
 module TwitterCldr
   module Shared
     autoload :Bidi,                   'twitter_cldr/shared/bidi'
-    autoload :BreakIterator,          'twitter_cldr/shared/break_iterator'
     autoload :Calendar,               'twitter_cldr/shared/calendar'
     autoload :Casefolder,             'twitter_cldr/shared/casefolder'
+    autoload :Caser,                  'twitter_cldr/shared/caser'
     autoload :CodePoint,              'twitter_cldr/shared/code_point'
     autoload :Currencies,             'twitter_cldr/shared/currencies'
     autoload :LanguageCodes,          'twitter_cldr/shared/language_codes'
     autoload :Languages,              'twitter_cldr/shared/languages'
+    autoload :LikelySubtags,          'twitter_cldr/shared/likely_subtags'
+    autoload :Locale,                 'twitter_cldr/shared/locale'
     autoload :NumberingSystem,        'twitter_cldr/shared/numbering_system'
     autoload :Numbers,                'twitter_cldr/shared/numbers'
     autoload :PhoneCodes,             'twitter_cldr/shared/phone_codes'
     autoload :PostalCodeGenerator,    'twitter_cldr/shared/postal_code_generator'
     autoload :PostalCodes,            'twitter_cldr/shared/postal_codes'
+    autoload :Properties,             'twitter_cldr/shared/properties'
+    autoload :PropertiesDatabase,     'twitter_cldr/shared/properties_database'
+    autoload :PropertyNameAliases,    'twitter_cldr/shared/property_name_aliases'
+    autoload :PropertyValueAliases,   'twitter_cldr/shared/property_value_aliases'
+    autoload :PropertyNormalizer,     'twitter_cldr/shared/property_normalizer'
+    autoload :PropertySet,            'twitter_cldr/shared/property_set'
     autoload :Territories,            'twitter_cldr/shared/territories'
     autoload :TerritoriesContainment, 'twitter_cldr/shared/territories_containment'
     autoload :Territory,              'twitter_cldr/shared/territory'

data/lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb

@@ -60,4 +60,4 @@ module TwitterCldr
 
     end
   end
-end
+end

data/lib/twitter_cldr/tokenizers/token.rb

@@ -15,7 +15,7 @@ module TwitterCldr
       end
 
       def to_hash
-        { :value => @value, :type => @type }
+        { value: @value, type: @type }
       end
 
       def to_s

data/lib/twitter_cldr/tokenizers/tokenizer.rb

@@ -49,7 +49,7 @@ module TwitterCldr
           Regexp.compile(
             tokenizers.map do |tokenizer|
               tokenizer.custom_splitter.source
-            end.join("|")
+            end.join("|"), nil, 'u'
           )
         end
 
@@ -79,17 +79,19 @@ module TwitterCldr
             recognizer.recognizes?(token_text)
           end
 
-          if recognizer.token_type == :composite
-            content = token_text.match(recognizer.content)[1]
-            ret << CompositeToken.new(tokenize(content))
-          else
-            cleaned_text = recognizer.clean(token_text)
-
-            if (remove_empty_entries && cleaned_text.size > 0) || !remove_empty_entries
-              ret << Token.new(
-                :value => cleaned_text,
-                :type => recognizer.token_type
-              )
+          if recognizer
+            if recognizer.token_type == :composite
+              content = token_text.match(recognizer.content)[1]
+              ret << CompositeToken.new(tokenize(content))
+            else
+              cleaned_text = recognizer.clean(token_text)
+
+              if (remove_empty_entries && cleaned_text.size > 0) || !remove_empty_entries
+                ret << Token.new(
+                  value: cleaned_text,
+                  type: recognizer.token_type
+                )
+              end
             end
           end
 
@@ -112,4 +114,4 @@ module TwitterCldr
 
     end
   end
-end
+end

data/lib/twitter_cldr/tokenizers/unicode_regex/unicode_regex_tokenizer.rb

@@ -21,8 +21,8 @@ module TwitterCldr
           recognizers = [
             # The variable name can contain letters and digits, but must start with a letter.
             TokenRecognizer.new(:variable, /\$\w[\w\d]*/),
-            TokenRecognizer.new(:character_set, /\[:\w+:\]|\\p\{[\w=]+\}/),  # [:Lu:] or \p{Lu} or \p{Sentence_Break=CF}
-            TokenRecognizer.new(:negated_character_set, /\[:\^\w+:\]|\\P\{[\w=]+\}/),  #[:^Lu:] or \P{Lu}
+            TokenRecognizer.new(:character_set, /\[:[\w\s=]+:\]|\\p\{[\w\s=]+\}/),  # [:Lu:] or \p{Lu} or \p{Sentence_Break=CF}
+            TokenRecognizer.new(:negated_character_set, /\[:\^[\w\s=]+:\]|\\P\{[\w\s=]+\}/),  #[:^Lu:] or \P{Lu}
             TokenRecognizer.new(:unicode_char, /\\u\{?[a-fA-F0-9]{1,6}\}?/),
             TokenRecognizer.new(:multichar_string, /\{\w+\}/u),
 
@@ -38,7 +38,7 @@ module TwitterCldr
             TokenRecognizer.new(:open_bracket, /\[/),
             TokenRecognizer.new(:close_bracket, /\]/),
 
-            TokenRecognizer.new(:string, //) do |val|
+            TokenRecognizer.new(:string, //u) do |val|
               val == " " ? val : val.strip
             end
           ]
@@ -49,4 +49,4 @@ module TwitterCldr
 
     end
   end
-end
+end

data/lib/twitter_cldr/utils/file_system_trie.rb

@@ -0,0 +1,145 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'yaml'
+require 'fileutils'
+
+module TwitterCldr
+  module Utils
+
+    class FileSystemTrie
+      VALUE_FILE = 'value.dump'
+
+      attr_reader :path_root
+
+      def initialize(path_root, root = Node.new)
+        @path_root = path_root
+        @root = root
+      end
+
+      def empty?
+        !@root.has_children?
+      end
+
+      def add(key, value)
+        store(key, value, false)
+      end
+
+      def set(key, value)
+        store(key, value)
+      end
+
+      def get(key)
+        node = get_node(key)
+        node && node.value
+      end
+
+      def get_node(key)
+        traverse(key) do |node, key_element|
+          return unless node
+          node.child(key_element)
+        end
+      end
+
+      # to prevent printing of a possibly huge children list in the IRB
+      alias_method :inspect, :to_s
+
+      private
+
+      def store(key, value, override = true)
+        final = store_p(key)
+
+        if final.value.nil? || override
+          final.value = value
+
+          path = File.join(path_root, *key, VALUE_FILE)
+          File.write(path, Marshal.dump(value))
+        end
+      end
+
+      def store_p(key)
+        current_path = path_root
+
+        traverse(key) do |node, key_element|
+          current_path = File.join(current_path, key_element)
+          mkdir(current_path)
+          node.child(key_element) || node.set_child(key_element, Node.new)
+        end
+      end
+
+      def traverse(key)
+        current_path = path_root
+
+        key.inject(@root) do |node, key_element|
+          next unless node
+          next unless key_element
+          current_path = File.join(current_path, key_element)
+          fill_in_path(current_path, key_element, node)
+          fill_in_value(current_path, key_element, node)
+          yield node, key_element if block_given?
+        end
+      end
+
+      def fill_in_path(current_path, key_element, parent)
+        if File.exist?(current_path)
+          unless parent.child(key_element)
+            parent.set_child(key_element, Node.new)
+          end
+        end
+      end
+
+      def fill_in_value(current_path, key_element, parent)
+        value_file = File.join(current_path, VALUE_FILE)
+        child = parent.child(key_element)
+
+        if File.exist?(value_file) && child && !child.value
+          parent.child(key_element).value = ::Marshal.load(
+            File.read(value_file)
+          )
+        end
+      end
+
+      def mkdir(path)
+        FileUtils.mkdir(path) unless File.exist?(path)
+      end
+
+      class Node
+
+        attr_accessor :value, :children
+
+        def initialize(value = nil, children = {})
+          @value    = value
+          @children = children
+        end
+
+        def child(key)
+          @children[key]
+        end
+
+        def set_child(key, child)
+          @children[key] = child
+        end
+
+        def has_children?
+          !@children.empty?
+        end
+
+        def each_key_and_child(&block)
+          @children.each(&block)
+        end
+
+        def keys
+          @children.keys
+        end
+
+        def to_trie
+          Trie.new(self.class.new(nil, @children)).lock
+        end
+
+      end
+    end
+
+  end
+end