twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/resources.rb

@@ -5,27 +5,26 @@
 
 module TwitterCldr
   module Resources
-    autoload :Uli,                                'twitter_cldr/resources/uli'
-    autoload :UnicodeImporter,                    'twitter_cldr/resources/unicode_importer'
-    autoload :IcuBasedImporter,                   'twitter_cldr/resources/icu_based_importer'
-    autoload :CanonicalCompositionsUpdater,       'twitter_cldr/resources/canonical_compositions_updater'
-    autoload :CollationTriesDumper,               'twitter_cldr/resources/collation_tries_dumper'
-    autoload :CompositionExclusionsImporter,      'twitter_cldr/resources/composition_exclusions_importer'
-    autoload :CurrenciesImporter,                 'twitter_cldr/resources/currencies_importer'
-    autoload :CustomLocalesResourcesImporter,     'twitter_cldr/resources/custom_locales_resources_importer'
-    autoload :LanguageCodesImporter,              'twitter_cldr/resources/language_codes_importer'
-    autoload :Loader,                             'twitter_cldr/resources/loader'
-    autoload :LocalesResourcesImporter,           'twitter_cldr/resources/locales_resources_importer'
-    autoload :PhoneCodesImporter,                 'twitter_cldr/resources/phone_codes_importer'
-    autoload :PostalCodesImporter,                'twitter_cldr/resources/postal_codes_importer'
-    autoload :TailoringImporter,                  'twitter_cldr/resources/tailoring_importer'
-    autoload :UnicodeDataImporter,                'twitter_cldr/resources/unicode_data_importer'
-    autoload :UnicodePropertiesImporter,          'twitter_cldr/resources/unicode_properties_importer'
-    autoload :BidiTestImporter,                   'twitter_cldr/resources/bidi_test_importer'
-    autoload :NormalizationQuickCheckImporter,    'twitter_cldr/resources/normalization_quick_check_importer'
-    autoload :RbnfTestImporter,                   'twitter_cldr/resources/rbnf_test_importer'
-    autoload :ReadmeRenderer,                     'twitter_cldr/resources/readme_renderer'
-    autoload :CasefolderClassGenerator,           'twitter_cldr/resources/casefolder_class_generator'
-    autoload :RegexpAstGenerator,                 'twitter_cldr/resources/regexp_ast_generator'
+    autoload :BidiTestImporter,               'twitter_cldr/resources/bidi_test_importer'
+    autoload :CasefolderClassGenerator,       'twitter_cldr/resources/casefolder_class_generator'
+    autoload :CollationTriesDumper,           'twitter_cldr/resources/collation_tries_dumper'
+    autoload :CurrenciesImporter,             'twitter_cldr/resources/currencies_importer'
+    autoload :CustomLocalesResourcesImporter, 'twitter_cldr/resources/custom_locales_resources_importer'
+    autoload :IcuBasedImporter,               'twitter_cldr/resources/icu_based_importer'
+    autoload :LanguageCodesImporter,          'twitter_cldr/resources/language_codes_importer'
+    autoload :Loader,                         'twitter_cldr/resources/loader'
+    autoload :LocalesResourcesImporter,       'twitter_cldr/resources/locales_resources_importer'
+    autoload :PhoneCodesImporter,             'twitter_cldr/resources/phone_codes_importer'
+    autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
+    autoload :Properties,                     'twitter_cldr/resources/properties'
+    autoload :RbnfTestImporter,               'twitter_cldr/resources/rbnf_test_importer'
+    autoload :ReadmeRenderer,                 'twitter_cldr/resources/readme_renderer'
+    autoload :RegexpAstGenerator,             'twitter_cldr/resources/regexp_ast_generator'
+    autoload :SegmentTestsImporter,           'twitter_cldr/resources/segment_tests_importer'
+    autoload :TailoringImporter,              'twitter_cldr/resources/tailoring_importer'
+    autoload :UnicodeDataImporter,            'twitter_cldr/resources/unicode_data_importer'
+    autoload :UnicodeImporter,                'twitter_cldr/resources/unicode_importer'
+    autoload :UnicodePropertyAliasesImporter, 'twitter_cldr/resources/unicode_property_aliases_importer'
+    autoload :Uli,                            'twitter_cldr/resources/uli'
   end
 end

data/lib/twitter_cldr/segmentation/break_iterator.rb

@@ -0,0 +1,54 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class BreakIterator
+
+      attr_reader :locale, :options
+
+      def initialize(locale = TwitterCldr.locale, options = {})
+        @locale = locale
+        @options = options
+      end
+
+      def each_sentence(str, &block)
+        rule_set = rule_set_for('sentence')
+        each_boundary(rule_set, str, &block)
+      end
+
+      def each_word(str, &block)
+        rule_set = rule_set_for('word')
+        each_boundary(rule_set, str, &block)
+      end
+
+      def each_grapheme_cluster(str, &block)
+        raise NotImplementedError,
+          "Grapheme segmentation is not currently supported."
+      end
+
+      def each_line(str, &block)
+        raise NotImplementedError,
+          "Line segmentation is not currently supported."
+      end
+
+      private
+
+      def each_boundary(rule_set, str)
+        if block_given?
+          rule_set.each_boundary(str).each_cons(2) do |start, stop|
+            yield str[start...stop]
+          end
+        else
+          to_enum(__method__, rule_set, str)
+        end
+      end
+
+      def rule_set_for(boundary_type)
+        RuleSet.load(locale, boundary_type, options)
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/cursor.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class Cursor
+      attr_reader :text, :position, :match_cache
+
+      def initialize(text)
+        @text = text
+        reset
+      end
+
+      def advance(amount = 1)
+        @position += amount
+      end
+
+      def reset
+        @position = 0
+        @match_cache = {}
+      end
+
+      def eof?
+        position >= text.size
+      end
+
+      def eos?
+        position >= text.size - 1
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/parser.rb

@@ -0,0 +1,71 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class Parser
+
+      def parse(text, options = {})
+        left_str, boundary_symbol_str, right_str = text.split(/([÷×])/)
+        boundary_symbol = boundary_symbol_for(boundary_symbol_str)
+        left = compile_token_list(tokenize_regex(left_str || ''), options)
+        right = compile_token_list(tokenize_regex(right_str || ''), options)
+        klass = class_for(boundary_symbol)
+        klass.new(left, right)
+      end
+
+      def tokenize_regex(text)
+        regex_tokenizer.tokenize(text).reject do |token|
+          token.value.strip.empty?
+        end
+      end
+
+      private
+
+      def boundary_symbol_for(str)
+        case str
+          when '÷' then :break
+          when '×' then :no_break
+        end
+      end
+
+      def class_for(boundary_symbol)
+        case boundary_symbol
+          when :break
+            BreakRule
+          when :no_break
+            NoBreakRule
+        end
+      end
+
+      def compile_token_list(token_list, options)
+        if token_list.empty?
+          TwitterCldr::Shared::UnicodeRegex.compile('')
+        else
+          parse_regex(token_list, options)
+        end
+      end
+
+      def parse_regex(tokens, options)
+        unless tokens.empty?
+          TwitterCldr::Shared::UnicodeRegex.new(
+            regex_parser.parse(tokens, options), 'm'
+          )
+        end
+      end
+
+      def regex_tokenizer
+        @tokenizer ||=
+          TwitterCldr::Tokenizers::UnicodeRegexTokenizer.new
+      end
+
+      def regex_parser
+        @regex_parser ||=
+          TwitterCldr::Parsers::UnicodeRegexParser.new
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/rule.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    RuleMatchData = Struct.new(
+      :rule, :boundary_offset, :boundary_position
+    )
+
+    class Rule
+
+      attr_reader :left, :right
+      attr_accessor :string, :id
+
+      def initialize(left, right)
+        @left = left
+        @right = right
+      end
+
+      def match(cursor)
+        left_match = match_side(left, cursor.text, cursor.position)
+        return nil unless left_match
+        left_match_offset = offset(left_match, cursor.position)
+
+        right_match = match_side(right, cursor.text, left_match_offset.last)
+        return nil unless right_match
+        right_match_offset = offset(right_match, left_match_offset.last)
+
+        offset = [left_match_offset.first, right_match_offset.last]
+        position = left_match_offset.last
+
+        RuleMatchData.new(self, offset, position)
+      end
+
+      private
+
+      def offset(match, default)
+        if match
+          match.offset(0)
+        else
+          [default, default]
+        end
+      end
+
+      def match_side(side, text, position)
+        if side
+          side_match = side.match(text, position)
+
+          if side_match && side_match.begin(0) == position
+            side_match
+          end
+        end
+      end
+    end
+
+    class BreakRule < Rule
+      def boundary_symbol
+        :break
+      end
+
+      def break?
+        true
+      end
+    end
+
+    class NoBreakRule < Rule
+      def boundary_symbol
+        :no_break
+      end
+
+      def break?
+        false
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/segmentation/rule_set.rb

@@ -0,0 +1,116 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class RuleSet
+
+      class << self
+        def load(*args)
+          RuleSetBuilder.load(*args)
+        end
+      end
+
+      attr_reader :locale, :rules, :boundary_type
+      attr_accessor :use_uli_exceptions
+
+      alias_method :use_uli_exceptions?, :use_uli_exceptions
+
+      def initialize(locale, rules, boundary_type, options)
+        @locale = locale
+        @rules = rules
+        @boundary_type = boundary_type
+        @use_uli_exceptions = options.fetch(
+          :use_uli_exceptions, false
+        )
+      end
+
+      def each_boundary(str)
+        if block_given?
+          cursor = Cursor.new(str)
+          last_boundary = 0
+
+          # implicit start of text boundary
+          yield 0
+
+          until cursor.eof?
+            match = find_match(cursor)
+            rule = match.rule
+
+            if rule.break?
+              yield match.boundary_position
+              last_boundary = match.boundary_position
+            end
+
+            if match.boundary_position == cursor.position
+              cursor.advance
+            else
+              cursor.advance(
+                match.boundary_position - cursor.position
+              )
+            end
+          end
+
+          # implicit end of text boundary
+          yield str.size unless last_boundary == str.size
+        else
+          to_enum(__method__, str)
+        end
+      end
+
+      private
+
+      def each_rule(&block)
+        if block_given?
+          if use_uli_exceptions? && supports_exceptions?
+            yield exception_rule
+          end
+
+          rules.each(&block)
+        else
+          to_enum(__method__)
+        end
+      end
+
+      def exception_rule
+        @exception_rule ||= RuleSetBuilder.exception_rule_for(
+          locale, boundary_type
+        )
+      end
+
+      def supports_exceptions?
+        boundary_type == 'sentence'
+      end
+
+      def find_match(cursor)
+        match = find_cached_match(cursor)
+
+        match || if cursor.eos?
+          RuleSetBuilder.implicit_end_of_text_rule.match(cursor)
+        else
+          RuleSetBuilder.implicit_final_rule.match(cursor)
+        end
+      end
+
+      def find_cached_match(cursor)
+        cursor.match_cache.fetch(cursor.position) do
+          matches = match_all(cursor)
+          matches.each do |m|
+            cursor.match_cache[m.boundary_position - 1] = m
+          end
+          matches.first
+        end
+      end
+
+      def match_all(cursor)
+        each_rule.each_with_object([]) do |rule, ret|
+          if match = rule.match(cursor)
+            ret << match
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/rule_set_builder.rb

@@ -0,0 +1,142 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class RuleSetBuilder
+
+      class << self
+        def load(locale, boundary_type, options = {})
+          rules = compile_rules_for(boundary_type)
+          RuleSet.new(locale, rules, boundary_type, options)
+        end
+
+        # See the comment above exceptions_for. Basically, we only support exceptions
+        # for the "sentence" boundary type since the ULI JSON data doesn't distinguish
+        # between boundary types.
+        def exception_rule_for(locale, boundary_type)
+          cache_key = TwitterCldr::Utils.compute_cache_key(locale, boundary_type)
+          exceptions_cache[cache_key] ||= begin
+            exceptions = exceptions_for(locale, boundary_type)
+            regex_contents = exceptions.map { |exc| Regexp.escape(exc) }.join("|")
+            parse("(?:#{regex_contents}) ×", nil).tap do |rule|
+              rule.id = 0
+            end
+          end
+        end
+
+        # The implicit final rule is always "Any ÷ Any"
+        def implicit_final_rule
+          @implicit_final_rule ||=
+            parse('. ÷ .', nil).tap do |rule|
+              rule.id = 9999
+            end
+        end
+
+        # The implicit initial rules are always "start-of-text ÷"
+        # and "÷ end-of-text". We don't need the start-of-text one.
+        def implicit_end_of_text_rule
+          @implicit_end_of_text_rule ||=
+            parse('.\z ÷', nil).tap do |rule|
+              rule.id = 9998
+            end
+        end
+
+        private
+
+        # The boundary_type param is not currently used since the ULI JSON resource that
+        # exceptions are generated from does not distinguish between boundary types. The
+        # XML version does, however, so the JSON will hopefully catch up at some point and
+        # we can make use of this second parameter. For the time being, compile_exception_rule_for
+        # (which calls this function) assumes a "sentence" boundary type.
+        def exceptions_for(locale, boundary_type)
+          exceptions_resource_cache[locale] ||= begin
+            TwitterCldr.get_resource('uli', 'segments', locale)[locale][:exceptions]
+          rescue Resources::ResourceLoadError
+            []
+          end
+        end
+
+        def boundary_name_for(str)
+          str.gsub(/(?:^|\_)([A-Za-z])/) { |s| $1.upcase } + 'Break'
+        end
+
+        # tokenizes and parses rules from segment_root
+        def compile_rules_for(boundary_type)
+          rule_cache[boundary_type] ||= begin
+            boundary_name = boundary_name_for(boundary_type)
+            boundary_data = resource_for(boundary_name)
+            symbol_table = symbol_table_for(boundary_data)
+            rules_for(boundary_data, symbol_table)
+          end
+        end
+
+        def symbol_table_for(boundary_data)
+          table = TwitterCldr::Parsers::SymbolTable.new
+          boundary_data[:variables].each do |variable|
+            id = variable[:id].to_s
+            tokens = segmentation_parser.tokenize_regex(variable[:value])
+            # note: variables can be redefined (add replaces if key already exists)
+            table.add(id, resolve_symbols(tokens, table))
+          end
+          table
+        end
+
+        def resolve_symbols(tokens, symbol_table)
+          tokens.inject([]) do |ret, token|
+            if token.type == :variable
+              ret += symbol_table.fetch(token.value)
+            else
+              ret << token
+            end
+            ret
+          end
+        end
+
+        def rules_for(boundary_data, symbol_table)
+          boundary_data[:rules].map do |rule|
+            r = parse(rule[:value], symbol_table)
+            r.string = rule[:value]
+            r.id = rule[:id]
+            r
+          end
+        end
+
+        def parse(text, symbol_table)
+          segmentation_parser.parse(
+            text, { symbol_table: symbol_table }
+          )
+        end
+
+        def resource_for(boundary_name)
+          root_resource[:segments][boundary_name.to_sym]
+        end
+
+        def segmentation_parser
+          @segmentation_parser ||= Segmentation::Parser.new
+        end
+
+        def root_resource
+          @root_resource ||= TwitterCldr.get_resource(
+            'shared', 'segments', 'segments_root'
+          )
+        end
+
+        def rule_cache
+          @rule_cache ||= {}
+        end
+
+        def exceptions_resource_cache
+          @exceptions_resource_cache ||= {}
+        end
+
+        def exceptions_cache
+          @exceptions_cache ||= {}
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/segmentation.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    autoload :BreakIterator,  'twitter_cldr/segmentation/break_iterator'
+    autoload :BreakRule,      'twitter_cldr/segmentation/rule'
+    autoload :Cursor,         'twitter_cldr/segmentation/cursor'
+    autoload :NoBreakRule,    'twitter_cldr/segmentation/rule'
+    autoload :Parser,         'twitter_cldr/segmentation/parser'
+    autoload :Rule,           'twitter_cldr/segmentation/rule'
+    autoload :RuleSet,        'twitter_cldr/segmentation/rule_set'
+    autoload :RuleSetBuilder, 'twitter_cldr/segmentation/rule_set_builder'
+  end
+end

data/lib/twitter_cldr/shared/bidi.rb

@@ -5,7 +5,7 @@
 
 # This code was adapted from GNU Classpath, but modified significantly.  Ordinarily, derivatives are
 # treated as falling under the same license as the original source, but classpath comes with the
-# following exception: 
+# following exception:
 #
 # "As a special exception, the copyright holders of this library give you
 # permission to link this library with independent modules to produce an
@@ -35,18 +35,18 @@ module TwitterCldr
       class << self
         def from_string(str, options = {})
           string_arr = str.unpack("U*")
-          Bidi.new(options.merge(:types => compute_types(string_arr), :string_arr => string_arr))
+          Bidi.new(options.merge(types: compute_types(string_arr), string_arr: string_arr))
         end
 
         def from_type_array(types, options = {})
-          Bidi.new(options.merge(:types => types))
+          Bidi.new(options.merge(types: types))
         end
 
         protected
 
         def compute_types(arr)
           arr.map do |code_point|
-            TwitterCldr::Shared::CodePoint.find(code_point).bidi_class.to_sym
+            TwitterCldr::Shared::CodePoint.get(code_point).bidi_class.to_sym
           end
         end
       end

data/lib/twitter_cldr/shared/calendar.rb

@@ -14,20 +14,20 @@ module TwitterCldr
       ERAS_NAMES_FORMS = [:abbr, :name]
 
       DATETIME_METHOD_MAP = {
-        :year_of_week_of_year => :year,
-        :quarter_stand_alone => :quarter,
-        :month_stand_alone => :month,
-        :day_of_month => :day,
-        :day_of_week_in_month => :day,
-        :weekday_local => :weekday,
-        :weekday_local_stand_alone => :weekday,
-        :second_fraction => :second,
-        :timezone_generic_non_location => :timezone,
-        :timezone_metazone => :timezone
+        year_of_week_of_year: :year,
+        quarter_stand_alone: :quarter,
+        month_stand_alone: :month,
+        day_of_month: :day,
+        day_of_week_in_month: :day,
+        weekday_local: :weekday,
+        weekday_local_stand_alone: :weekday,
+        second_fraction: :second,
+        timezone_generic_non_location: :timezone,
+        timezone_metazone: :timezone
       }
 
       REDIRECT_CONVERSIONS = {
-        :dayPeriods => :periods
+        dayPeriods: :periods
       }
 
       attr_reader :locale, :calendar_type