twitter_cldr 3.6.0 → 4.0.0

data/lib/twitter_cldr/segmentation/break_iterator.rb

@@ -39,7 +39,7 @@ module TwitterCldr
       def each_boundary(rule_set, str)
         if block_given?
           rule_set.each_boundary(str).each_cons(2) do |start, stop|
-            yield str[start...stop]
+            yield str[start...stop], start, stop
           end
         else
           to_enum(__method__, rule_set, str)

data/lib/twitter_cldr/shared.rb

@@ -11,6 +11,7 @@ module TwitterCldr
     autoload :Caser,                  'twitter_cldr/shared/caser'
     autoload :CodePoint,              'twitter_cldr/shared/code_point'
     autoload :Currencies,             'twitter_cldr/shared/currencies'
+    autoload :Hyphenator,             'twitter_cldr/shared/hyphenator'
     autoload :LanguageCodes,          'twitter_cldr/shared/language_codes'
     autoload :Languages,              'twitter_cldr/shared/languages'
     autoload :LikelySubtags,          'twitter_cldr/shared/likely_subtags'

data/lib/twitter_cldr/shared/hyphenator.rb

@@ -0,0 +1,188 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Documentation: https://github.com/hunspell/hyphen/blob/21127cc8493a68d4fe9adbb71377b469b4f2b550/doc/tb87nemeth.pdf
+
+module TwitterCldr
+  module Shared
+    class Hyphenator
+      class UnsupportedLocaleError < StandardError; end
+
+      BASE_RESOURCE_PATH = %w(shared hyphenation)
+      DEFAULT_LEFT_HYPHEN_MIN = 2
+      DEFAULT_RIGHT_HYPHEN_MIN = 2
+      DEFAULT_NO_HYPHEN = %w(- ' ’)
+
+      class << self
+        def get(locale)
+          locale = find_supported_locale(locale)
+
+          unless locale
+            raise UnsupportedLocaleError,
+              "'#{locale}' is not a supported hyphenation locale"
+          end
+
+          cache[locale] ||= begin
+            resource = resource_for(locale)
+            new(resource[:rules], locale, resource[:options])
+          end
+        end
+
+        def supported_locale?(locale)
+          !!find_supported_locale(locale)
+        end
+
+        def supported_locales
+          @supported_locales ||= begin
+            absolute_resource_path = TwitterCldr.absolute_resource_path(
+              File.join(BASE_RESOURCE_PATH)
+            )
+
+            files = Dir.glob(File.join(absolute_resource_path, '*.yml'))
+            files.map { |f| File.basename(f).chomp('.yml') }
+          end
+        end
+
+        private
+
+        def find_supported_locale(locale)
+          maximized_locale = Locale.parse(locale.to_s).maximize
+
+          maximized_locale.permutations('-').find do |locale_candidate|
+            TwitterCldr.resource_exists?(
+              *BASE_RESOURCE_PATH, locale_candidate
+            )
+          end
+        end
+
+        def cache
+          @cache ||= {}
+        end
+
+        def resource_for(locale)
+          TwitterCldr.get_resource(*BASE_RESOURCE_PATH, locale)
+        end
+      end
+
+      attr_reader :rules, :locale, :options, :trie
+
+      def initialize(rules, locale, options)
+        @rules = rules
+        @locale = locale
+        @options = options
+        @trie = build_trie_from(rules)
+      end
+
+      # 0x00AD is a soft hyphen
+      def hyphenate(text, hyphen = "\u00AD")
+        each_chunk(text).to_a.join(hyphen)
+      end
+
+      def each_chunk(text)
+        if block_given?
+          last_pos = 0
+
+          each_position(text) do |pos|
+            yield text[last_pos...pos].tap { last_pos = pos }
+          end
+
+          if last_pos < text.size
+            yield text[last_pos..text.size]
+          end
+        else
+          to_enum(__method__, text)
+        end
+      end
+
+      def each_position(text)
+        if block_given?
+          text = ".#{text}."
+          break_weights = break_weights_for(text)
+
+          left = left_hyphen_min
+          right = text.size - right_hyphen_min - 2
+
+          (left...right).each do |idx|
+            yield idx if break_weights[idx].odd?
+          end
+        else
+          to_enum(__method__, text)
+        end
+      end
+
+      private
+
+      def break_weights_for(text)
+        break_weights = Array.new(text.size - 1, 0)
+
+        text.each_char.with_index do |char, idx|
+          subtrie = trie.root
+          counter = idx
+
+          while subtrie
+            subtrie = subtrie.child(text[counter])
+            counter += 1
+
+            if subtrie && subtrie.has_value?
+              update_break_weights(subtrie.value, break_weights, idx)
+            end
+          end
+        end
+
+        remove_illegal_hyphens(break_weights, text)
+      end
+
+      def update_break_weights(pattern, break_weights, start_idx)
+        pattern_idx = 0
+
+        pattern.each_char do |segment|
+          if segment =~ /\d/
+            int_seg = segment.to_i
+            idx = (start_idx + pattern_idx) - 1
+            break if idx >= break_weights.size
+
+            break_weights[idx] = if break_weights[idx] > int_seg
+              break_weights[idx]
+            else
+              int_seg
+            end
+          else
+            pattern_idx += 1
+          end
+        end
+      end
+
+      def remove_illegal_hyphens(break_weights, text)
+        break_weights.map.with_index do |break_weight, idx|
+          next break_weight if idx.zero?
+          next 0 if no_hyphen.include?(text[idx - 1])
+          break_weight
+        end
+      end
+
+      def left_hyphen_min
+        @left_hyphen_min ||=
+          options.fetch(:lefthyphenmin, DEFAULT_LEFT_HYPHEN_MIN)
+      end
+
+      def right_hyphen_min
+        @right_hyphen_min ||=
+          options.fetch(:righthyphenmin, DEFAULT_RIGHT_HYPHEN_MIN)
+      end
+
+      def no_hyphen
+        @no_hyphen ||= options.fetch(:nohyphen, DEFAULT_NO_HYPHEN)
+      end
+
+      def build_trie_from(rules)
+        TwitterCldr::Utils::Trie.new.tap do |trie|
+          rules.each do |rule|
+            trie.add(rule.gsub(/\d/, '').each_char, rule)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twitter_cldr/shared/likely_subtags.rb

@@ -17,6 +17,8 @@ module TwitterCldr
 
         private
 
+        # http://unicode.org/reports/tr35/#Likely_Subtags
+        #
         # Try each of the following in order (where the fields exist). The
         # notation field³ means field¹ if it exists, otherwise field².
         #
@@ -31,11 +33,15 @@ module TwitterCldr
         #
         # 4. Lookup language¹. If in the table, return language² _ script³ _
         #    region³ + variants.
+        #
+        # 5. Lookup und_script¹ and return if exists
+        #
         def lookup(locale)
           first_lookup(locale) ||
             second_lookup(locale) ||
             third_lookup(locale) ||
             fourth_lookup(locale) ||
+            fifth_lookup(locale) ||
             raise(
               UnrecognizedSubtagsError,
               "couldn't find matching subtags for #{locale}"
@@ -44,7 +50,7 @@ module TwitterCldr
 
         def first_lookup(locale)
           if locale.language && locale.script && locale.region
-            code = [locale.language, locale.script, locale.region].join('_')
+            code = [locale.language, locale.abbreviated_script, locale.region].join('_')
 
             if replacement = subtags_resource[code.to_sym]
               language2, script2, region2 = Locale.split(replacement)
@@ -55,7 +61,7 @@ module TwitterCldr
 
         def second_lookup(locale)
           if locale.language && locale.script
-            code = [locale.language, locale.script].join('_')
+            code = [locale.language, locale.abbreviated_script].join('_')
 
             if replacement = subtags_resource[code.to_sym]
               language2, script2, region2 = Locale.split(replacement)
@@ -93,6 +99,16 @@ module TwitterCldr
           end
         end
 
+        def fifth_lookup(locale)
+          if locale.script
+            code = ['und', locale.abbreviated_script].join('_')
+
+            if replacement = subtags_resource[code.to_sym]
+              Locale.parse(replacement)
+            end
+          end
+        end
+
         def subtags_resource
           @subtags_resource ||=
             TwitterCldr.get_resource('shared', 'likely_subtags')[:subtags]

data/lib/twitter_cldr/shared/locale.rb

@@ -26,7 +26,7 @@ module TwitterCldr
         # 5. Get the components of the cleaned-up tag (language¹, script¹, and
         #    region¹), plus any variants if they exist (including keywords).
         def parse(locale_text)
-          locale_text = locale_text.strip
+          locale_text = locale_text.to_s.strip
           return Locale.new(locale_text) if grandfathered?(locale_text)
 
           normalize(locale_text).tap do |locale|
@@ -123,7 +123,8 @@ module TwitterCldr
 
         def script?(subtag)
           subtag = normalize_subtag(subtag, :script)
-          scripts.include?(subtag)
+          scripts.include?(subtag) ||
+            !!PropertyValueAliases.long_alias_for('sc', subtag)
         end
 
         def region?(subtag)
@@ -168,14 +169,8 @@ module TwitterCldr
         end
 
         def remove_placeholder_tags(locale)
-          if locale.script == 'Zzzz'
-            locale.script = nil
-          end
-
-          if locale.region == 'ZZ'
-            locale.region = nil
-          end
-
+          locale.script = nil if locale.script == 'Zzzz'
+          locale.region = nil if locale.region == 'ZZ'
           locale.language ||= 'und'
         end
 
@@ -224,6 +219,10 @@ module TwitterCldr
         @full_script ||= PropertyValueAliases.long_alias_for('sc', script) || script
       end
 
+      def abbreviated_script
+        @short_script ||= PropertyValueAliases.abbreviated_alias_for('sc', script) || script
+      end
+
       def maximize
         if Locale.grandfathered?(to_s)
           self
@@ -247,6 +246,17 @@ module TwitterCldr
         ([language, script, region] + variants).compact
       end
 
+      def permutations(delimiter = '_')
+        perms = [
+          [language, script, region].compact.join(delimiter),
+          [language, script].compact.join(delimiter),
+          [language, region].compact.join(delimiter),
+          language,
+        ]
+
+        perms.uniq
+      end
+
     end
   end
 end

data/lib/twitter_cldr/transforms.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Transforms
+    autoload :CommentRule,       'twitter_cldr/transforms/comment_rule'
+    autoload :Conversions,       'twitter_cldr/transforms/conversions'
+    autoload :ConversionRuleSet, 'twitter_cldr/transforms/conversion_rule_set'
+    autoload :Cursor,            'twitter_cldr/transforms/cursor'
+    autoload :Filters,           'twitter_cldr/transforms/filters'
+    autoload :Locale,            'twitter_cldr/transforms/transformer'
+    autoload :Rule,              'twitter_cldr/transforms/rule'
+    autoload :RuleMatch,         'twitter_cldr/transforms/rule_match'
+    autoload :RuleSet,           'twitter_cldr/transforms/rule_set'
+    autoload :Tokenizer,         'twitter_cldr/transforms/tokenizer'
+    autoload :Transformer,       'twitter_cldr/transforms/transformer'
+    autoload :Transforms,        'twitter_cldr/transforms/transforms'
+    autoload :TransformId,       'twitter_cldr/transforms/transform_id'
+    autoload :Transliterator,    'twitter_cldr/transforms/transliterator'
+    autoload :VariableRule,      'twitter_cldr/transforms/variable_rule'
+  end
+end

data/lib/twitter_cldr/transforms/comment_rule.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'singleton'
+
+module TwitterCldr
+  module Transforms
+
+    class CommentRule < Rule
+      include Singleton
+
+      class << self
+        def parse(rule_text, symbol_table, index)
+          instance
+        end
+
+        def accepts?(rule_text)
+          rule_text.strip.start_with?('#')
+        end
+      end
+
+      def is_comment?
+        true
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/transforms/conversion_rule_set.rb

@@ -0,0 +1,155 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# http://unicode.org/reports/tr35/tr35-general.html#Transforms
+# http://unicode.org/cldr/utility/transform.jsp
+
+module TwitterCldr
+  module Transforms
+
+    class ConversionRuleSet
+      attr_reader :filter_rule, :inverse_filter_rule
+      attr_reader :rules, :rule_index
+
+      def initialize(filter_rule, inverse_filter_rule, rules)
+        @rules = rules
+        @filter_rule = filter_rule
+        @inverse_filter_rule = inverse_filter_rule
+        @rule_index = build_rule_index(rules)
+      end
+
+      def foward?
+        true
+      end
+
+      def backward?
+        false
+      end
+
+      def is_transform_rule?
+        false
+      end
+
+      def is_conversion_rule?
+        false
+      end
+
+      def invert
+        ConversionRuleSet.new(
+          inverse_filter_rule, filter_rule, inverted_rules
+        )
+      end
+
+      def apply_to(cursor)
+        until cursor.eos?
+          if filter_rule.matches?(cursor)
+            rule_match = find_matching_rule_at(cursor)
+
+            if rule_match
+              start = rule_match.start
+              stop = rule_match.stop
+              replacement = rule_match.replacement
+              cursor.text[start...stop] = replacement
+
+              cursor.advance(
+                replacement.size + rule_match.cursor_offset
+              )
+            else
+              cursor.advance
+            end
+          else
+            cursor.advance
+          end
+        end
+      end
+
+      private
+
+      def inverted_rules
+        @inverted_rules ||= begin
+          rules.each_with_object([]) do |rule, ret|
+            if rule.can_invert?
+              ret << rule.invert
+            end
+          end
+        end
+      end
+
+      def find_matching_rule_at(cursor)
+        indexed_match = find_matching_indexed_rule_at(cursor)
+        blank_key_match = find_matching_blank_key_rule_at(cursor)
+
+        if indexed_match
+          if blank_key_match
+            if blank_key_match < indexed_match
+              blank_key_match
+            else
+              indexed_match
+            end
+          else
+            indexed_match
+          end
+        else
+          blank_key_match
+        end
+      end
+
+      def find_matching_indexed_rule_at(cursor)
+        if rules = rule_index.get(cursor.index_values)
+          rules.each do |rule|
+            if side_match = rule.match(cursor)
+              return RuleMatch.new(rule, side_match)
+            end
+          end
+        end
+
+        nil
+      end
+
+      def find_matching_blank_key_rule_at(cursor)
+        if rules = rule_index.get([0])
+          rules.each do |rule|
+            if side_match = rule.match(cursor)
+              return RuleMatch.new(rule, side_match)
+            end
+          end
+        end
+
+        nil
+      end
+
+      def build_rule_index(rules)
+        TwitterCldr::Utils::Trie.new.tap do |trie|
+          rules.each_with_index do |rule, idx|
+            next unless rule.forward?
+
+            if rule.has_codepoints?
+              codepoints = rule.codepoints
+
+              if codepoints.size > 0
+                rule.codepoints.each do |codepoint|
+                  bytes = codepoint.chr('UTF-8').bytes
+
+                  if entry = trie.get(bytes)
+                    entry << rule
+                  else
+                    trie.add(bytes, [rule])
+                  end
+                end
+              end
+            else
+              if entry = trie.get([0])
+                entry << rule
+              else
+                trie.add([0], [rule])
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end