twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/parsers/segmentation_parser.rb

@@ -1,137 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Parsers
-
-    class SegmentationParser < Parser
-
-      RuleMatchData = Struct.new(:text, :boundary_offset)
-
-      class Rule
-        attr_accessor :string, :id
-      end
-
-      class BreakRule < Rule
-
-        attr_reader :left, :right
-
-        def initialize(left, right)
-          @left = left
-          @right = right
-        end
-
-        def match(str)
-          if left && left_match = left.match(str)
-            match_pos = left_match.offset(0).last
-
-            if right
-              if right_match = right.match(str[match_pos..-1])
-                RuleMatchData.new(
-                  left_match[0] + right_match[0],
-                  match_pos
-                )
-              end
-            else
-              RuleMatchData.new(str, str.size)
-            end
-          end
-        end
-
-        def boundary_symbol
-          :break
-        end
-
-      end
-
-      class NoBreakRule < Rule
-
-        attr_reader :regex
-
-        def initialize(regex)
-          @regex = regex
-        end
-
-        def match(str)
-          if match = regex.match(str)
-            RuleMatchData.new(match[0], match.offset(0).last)
-          end
-        end
-
-        def boundary_symbol
-          :no_break
-        end
-
-      end
-
-      private
-
-      def do_parse(options)
-        regex_token_lists = []
-        current_regex_tokens = []
-        boundary_symbol = nil
-
-        while current_token
-          case current_token.type
-            when :break, :no_break
-              boundary_symbol = current_token.type
-              regex_token_lists << current_regex_tokens
-              current_regex_tokens = []
-            else
-              current_regex_tokens << current_token
-          end
-
-          next_token(current_token.type)
-        end
-
-        regex_token_lists << current_regex_tokens
-
-        case boundary_symbol
-          when :break
-            BreakRule.new(
-              parse_regex(add_anchors(regex_token_lists[0]), options),
-              parse_regex(add_anchors(regex_token_lists[1]), options)
-            )
-          when :no_break
-            NoBreakRule.new(
-              parse_regex(add_anchors(regex_token_lists.flatten), options)
-            )
-        end
-      end
-
-      # only find matches from the beginning of the current string
-      def add_anchors(token_list)
-        token_list.insert(0, begin_token)
-      end
-
-      def begin_token
-        self.class.begin_token
-      end
-
-      def self.begin_token
-        @begin_token ||= TwitterCldr::Tokenizers::Token.new(
-          :type => :special_char, :value => "\\A"
-        )
-      end
-
-      def parse_regex(tokens, options)
-        unless tokens.empty?
-          TwitterCldr::Shared::UnicodeRegex.new(
-            regex_parser.parse(tokens, options)
-          )
-        end
-      end
-
-      def regex_parser
-        self.class.regex_parser
-      end
-
-      def self.regex_parser
-        @regex_parser ||= UnicodeRegexParser.new
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/resources/canonical_compositions_updater.rb

@@ -1,51 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Resources
-
-    class CanonicalCompositionsUpdater
-
-      CODE_POINT_MAX = 0x10FFFF
-
-      # Arguments:
-      #
-      #   output_path - output directory for generated YAML file
-      #
-      def initialize(output_path)
-        @output_path = output_path
-      end
-
-      def update
-        File.open(File.join(@output_path, 'canonical_compositions.yml'), 'w') do |output|
-          YAML.dump(generate_compositions, output)
-        end
-      end
-
-      private
-
-      def generate_compositions
-        (1..CODE_POINT_MAX).inject({}) do |memo, code_point|
-          code_point_data = TwitterCldr::Shared::CodePoint.find(code_point)
-
-          if code_point_data && !code_point_data.compatibility_decomposition? && code_point_data.decomposition && !code_point_data.decomposition.empty?
-            memo[code_point_data.decomposition] = code_point
-          end
-
-          log_progress(code_point, memo.size)
-
-          memo
-        end
-      end
-
-      def log_progress(code_point, compositions_count)
-        $stdout.write("\r#{(100.0 * code_point / CODE_POINT_MAX).round}% complete, found #{compositions_count} canonical compositions")
-        $stdout.write("\n") if code_point == CODE_POINT_MAX
-      end
-
-    end
-
-  end
-end

data/lib/twitter_cldr/resources/composition_exclusions_importer.rb

@@ -1,62 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'twitter_cldr/resources/download'
-
-module TwitterCldr
-  module Resources
-
-    class CompositionExclusionsImporter
-
-      COMPOSITION_EXCLUSIONS_URL   = 'http://www.unicode.org/Public/6.1.0/ucd/DerivedNormalizationProps.txt'
-      COMPOSITION_EXCLUSION_REGEXP = /^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+; Full_Composition_Exclusion #.*$/
-      TOTAL_CODE_POINTS_REGEXP     = /^# Total code points: (\d+)$/
-
-      # Arguments:
-      #
-      #   input_path  - path to DerivedNormalizationProps.txt file
-      #   output_path - output directory for generated YAML file
-      #
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
-
-      def import
-        File.open(File.join(@output_path, 'composition_exclusions.yml'), 'w') do |output|
-          YAML.dump(generate_composition_exclusions, output)
-        end
-      end
-
-      private
-
-      def generate_composition_exclusions
-        data      = File.open(composition_exclusions_file) { |file| file.read }
-        start_pos = data.index("# Derived Property: Full_Composition_Exclusion")
-        end_pos   = data.index(/^#\s=*$/, start_pos)
-        data      = data[start_pos..end_pos].split("\n")
-
-        expected_code_points_count = nil
-
-        result = data.inject([]) do |memo, line|
-          memo << ($1.hex..($2 || $1).hex)     if line =~ COMPOSITION_EXCLUSION_REGEXP
-          expected_code_points_count = $1.to_i if line =~ TOTAL_CODE_POINTS_REGEXP
-          memo
-        end
-
-        raise "Expected number of code points was not found." unless expected_code_points_count
-        code_points_count = result.map(&:count).inject(:+)
-        raise "Unexpected number of code points: expected - #{expected_code_points_count}, got - #{code_points_count}." unless code_points_count == expected_code_points_count
-
-        result
-      end
-
-      def composition_exclusions_file
-        TwitterCldr::Resources.download_if_necessary(@input_path, COMPOSITION_EXCLUSIONS_URL)
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/resources/normalization_quick_check_importer.rb

@@ -1,73 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'twitter_cldr/resources/download'
-
-module TwitterCldr
-  module Resources
-
-    class NormalizationQuickCheckImporter
-
-      PROPS_FILE_URL = "ftp://ftp.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt"
-
-      # Arguments:
-      #
-      #   input_path  - path to a directory containing DerivedNormalizationProps.txt
-      #   output_path - output directory for imported YAML files
-      #
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
-
-      def import
-        parse_props_file.each_pair do |algorithm, code_point_list|
-          File.open(File.join(@output_path, "#{algorithm.downcase}_quick_check.yml"), "w+") do |f|
-            f.write(YAML.dump(TwitterCldr::Utils::RangeSet.rangify(code_point_list)))
-          end
-        end
-      end
-
-      private
-
-      def parse_props_file
-        check_table = {}
-        cur_type = nil
-
-        File.open(props_file) do |input|
-          input.each_line do |line|
-            cur_type = nil if line =~ /=Maybe/
-            type = line.scan(/#\s*Property:\s*(NF[KDC]+)_Quick_Check/).flatten
-
-            if type.size > 0
-              cur_type = type.first
-              check_table[cur_type] = []
-            end
-
-            if check_table.size > 0 && line[0...1] != "#" && !line.strip.empty? && cur_type
-              start, finish = line.scan(/(\h+(\.\.\h+)?)/).first.first.split("..").map { |num| num.to_i(16) }
-
-              if finish
-                check_table[cur_type] += (start..finish).to_a
-              else
-                check_table[cur_type] << start
-              end
-            end
-
-            break if line =~ /={5,}/ && check_table.size >= 4 && check_table.all? { |key, val| val.size > 0 }
-          end
-        end
-
-        check_table
-      end
-
-      def props_file
-        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'DerivedNormalizationProps.txt'), PROPS_FILE_URL)
-      end
-
-    end
-
-  end
-end

data/lib/twitter_cldr/resources/unicode_properties_importer.rb

@@ -1,79 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'twitter_cldr/resources/download'
-
-module TwitterCldr
-  module Resources
-
-    class UnicodePropertiesImporter < UnicodeImporter
-
-      PROPERTIES_BASE_URL = 'ftp://ftp.unicode.org/Public/UCD/latest/ucd'
-      PROPERTIES = [
-        "auxiliary/SentenceBreakProperty",
-        "auxiliary/WordBreakProperty",
-        "LineBreak"
-      ]
-
-      # Arguments:
-      #
-      #   input_path  - path to a directory containing the various property files
-      #   output_path - output directory for imported YAML files
-      #
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
-
-      def import
-        FileUtils.mkdir_p(@output_path)
-
-        PROPERTIES.each do |property|
-          input_file = property_data_file("#{property}.txt")
-          output_file = File.join(@output_path, "#{fix_name(property)}.yml")
-
-          File.open(output_file, "w+") do |f|
-            f.write(
-              YAML.dump(
-                parse_standard_file(input_file).inject({}) do |ret, data|
-                  name = data[1].strip.to_sym
-                  ret[name] ||= []
-                  ret[name] += expand_range(data[0])
-                  ret
-                end.inject({}) do |ret, (key, data)|
-                  ret[key] = TwitterCldr::Utils::RangeSet.rangify(data)
-                  ret
-                end
-              )
-            )
-          end
-        end
-      end
-
-      private
-
-      def fix_name(str)
-        underscore(File.basename(str.gsub("Property", "")))
-      end
-
-      def underscore(str)
-        str.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
-      end
-
-      def expand_range(str)
-        initial, final = str.split("..")
-        (initial.to_i(16)..(final || initial).to_i(16)).to_a
-      end
-
-      def property_data_file(file)
-        TwitterCldr::Resources.download_if_necessary(
-          File.join(@input_path, File.basename(file)), File.join(PROPERTIES_BASE_URL, file)
-        )
-      end
-
-    end
-
-  end
-end

data/lib/twitter_cldr/shared/break_iterator.rb

@@ -1,213 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Shared
-    class BreakIterator
-
-      attr_reader :locale, :use_uli_exceptions
-
-      def initialize(locale = TwitterCldr.locale, options = {})
-        @use_uli_exceptions = !!options.fetch(:use_uli_exceptions, true)
-        @locale = locale
-      end
-
-      def each_sentence(str, &block)
-        each_boundary(str, "sentence", &block)
-      end
-
-      def each_word(str, &block)
-        raise NotImplementedError.new("Word segmentation is not currently supported.")
-      end
-
-      def each_line(str, &block)
-        raise NotImplementedError.new("Line segmentation is not currently supported.")
-      end
-
-      private
-
-      def boundary_name_for(str)
-        str.gsub(/(?:^|\_)([A-Za-z])/) { |s| $1.upcase } + "Break"
-      end
-
-      def each_boundary(str, boundary_type)
-        if block_given?
-          rules = compile_rules_for(locale, boundary_type)
-          match = nil
-          last_offset = 0
-          current_position = 0
-          search_str = str.dup
-
-          until search_str.size == 0
-            rule = rules.find { |rule| match = rule.match(search_str) }
-
-            if rule.boundary_symbol == :break
-              break_offset = current_position + match.boundary_offset
-              yield str[last_offset...break_offset]
-              last_offset = break_offset
-            end
-
-            search_str = search_str[match.boundary_offset..-1]
-            current_position += match.boundary_offset
-          end
-
-          if last_offset < (str.size - 1)
-            yield str[last_offset..-1]
-          end
-        else
-          to_enum(__method__, str, boundary_type)
-        end
-      end
-
-      # See the comment above exceptions_for. Basically, we only support exceptions
-      # for the "sentence" boundary type since the ULI JSON data doesn't distinguish
-      # between boundary types.
-      def compile_exception_rule_for(locale, boundary_type, boundary_name)
-        if boundary_type == "sentence"
-          cache_key = TwitterCldr::Utils.compute_cache_key(locale, boundary_type)
-          self.class.exceptions_cache[cache_key] ||= begin
-            exceptions = exceptions_for(locale, boundary_name)
-            regex_contents = exceptions.map { |exc| Regexp.escape(exc) }.join("|")
-            segmentation_parser.parse(
-              segmentation_tokenizer.tokenize("(?:#{regex_contents}) ×")
-            )
-          end
-        end
-      end
-
-      def self.exceptions_cache
-        @exceptions_cache ||= {}
-      end
-
-      # Grabs rules from segment_root, applies custom tailorings (our own, NOT from CLDR),
-      # and optionally integrates ULI exceptions.
-      def compile_rules_for(locale, boundary_type)
-        rules = self.class.rule_cache[boundary_type] ||= begin
-          boundary_name = boundary_name_for(boundary_type)
-          boundary_data = resource_for(boundary_name)
-          symbol_table = symbol_table_for(boundary_data)
-          root_rules = rules_for(boundary_data, symbol_table)
-
-          tailoring_boundary_data = tailoring_resource_for(locale, boundary_name)
-          tailoring_rules = rules_for(tailoring_boundary_data, symbol_table)
-          merge_rules(root_rules, tailoring_rules)
-        end
-
-        if use_uli_exceptions
-          exception_rule = compile_exception_rule_for(locale, boundary_type, boundary_name)
-          rules = rules.dup  # avoid modifying the cached rules
-          rules.insert(0, exception_rule)
-        end
-
-        rules
-      end
-
-      # replaces ruleset1's rules with rules with the same id from ruleset2
-      def merge_rules(ruleset1, ruleset2)
-        result = ruleset1.dup
-        ruleset2.each do |new_rule|
-          if existing_idx = result.find_index { |rule| rule.id == new_rule.id }
-            result[existing_idx] = new_rule
-          end
-        end
-        result
-      end
-
-      def self.rule_cache
-        @rule_cache ||= {}
-      end
-
-      def symbol_table_for(boundary_data)
-        table = TwitterCldr::Parsers::SymbolTable.new
-        boundary_data[:variables].each do |variable|
-          id = variable[:id].to_s
-          tokens = segmentation_tokenizer.tokenize(variable[:value])
-          # note: variables can be redefined (add replaces if key already exists)
-          table.add(id, resolve_symbols(tokens, table))
-        end
-        table
-      end
-
-      def resolve_symbols(tokens, symbol_table)
-        tokens.inject([]) do |ret, token|
-          if token.type == :variable
-            ret += symbol_table.fetch(token.value)
-          else
-            ret << token
-          end
-          ret
-        end
-      end
-
-      def rules_for(boundary_data, symbol_table)
-        boundary_data[:rules].map do |rule|
-          r = segmentation_parser.parse(
-            segmentation_tokenizer.tokenize(rule[:value]), {
-              :symbol_table => symbol_table
-            }
-          )
-
-          r.string = rule[:value]
-          r.id = rule[:id]
-          r
-        end
-      end
-
-      def self.segmentation_tokenizer
-        @segmentation_tokenizer ||= TwitterCldr::Tokenizers::SegmentationTokenizer.new
-      end
-
-      def segmentation_tokenizer
-        self.class.segmentation_tokenizer
-      end
-
-      def self.segmentation_parser
-        @segmentation_parser ||= TwitterCldr::Parsers::SegmentationParser.new
-      end
-
-      def segmentation_parser
-        self.class.segmentation_parser
-      end
-
-      def resource_for(boundary_name)
-        self.class.root_resource[:segments][boundary_name.to_sym]
-      end
-
-      def tailoring_resource_for(locale, boundary_name)
-        cache_key = TwitterCldr::Utils.compute_cache_key(locale, boundary_name)
-        self.class.tailoring_resource_cache[cache_key] ||= begin
-          res = TwitterCldr.get_resource("shared", "segments", "tailorings", locale)
-          res[locale][:segments][boundary_name.to_sym]
-        end
-      end
-
-      def self.tailoring_resource_cache
-        @tailoring_resource_cache ||= {}
-      end
-
-      def self.root_resource
-        @root_resource ||= TwitterCldr.get_resource("shared", "segments", "segments_root")
-      end
-
-      # The boundary_name param is not currently used since the ULI JSON resource that
-      # exceptions are generated from does not distinguish between boundary types. The
-      # XML version does, however, so the JSON will hopefully catch up at some point and
-      # we can make use of this second parameter. For the time being, compile_exception_rule_for
-      # (which calls this function) assumes a "sentence" boundary type.
-      def exceptions_for(locale, boundary_name)
-        self.class.exceptions_resource_cache[locale] ||= begin
-          TwitterCldr.get_resource("uli", "segments", locale)[locale][:exceptions]
-        rescue ArgumentError
-          []
-        end
-      end
-
-      def self.exceptions_resource_cache
-        @exceptions_resource_cache ||= {}
-      end
-
-    end
-  end
-end

data/lib/twitter_cldr/tokenizers/segmentation/segmentation_tokenizer.rb

@@ -1,39 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-module TwitterCldr
-  module Tokenizers
-    class SegmentationTokenizer
-
-      def tokenize(pattern)
-        # according to the spec, whitespace should be ignored
-        tokenizer.tokenize(pattern).reject do |token|
-          token.value.strip.empty?
-        end
-      end
-
-      private
-
-      def tokenizer
-        @tokenizer ||= begin
-          recognizers = [
-            TokenRecognizer.new(:break, /\303\267/u) do |val|     # ÷ character
-              val.strip
-            end,
-
-            TokenRecognizer.new(:no_break, /\303\227/u) do |val|  # × character
-              val.strip
-            end
-          ]
-
-          ur_tokenizer = UnicodeRegexTokenizer.new
-          ur_tokenizer.insert_before(:string, *recognizers)
-          ur_tokenizer
-        end
-      end
-
-    end
-  end
-end

data/resources/shared/segments/tailorings/en.yml

@@ -1,8 +0,0 @@
---- 
-:en:
-  :segments:
-    :SentenceBreak:
-      :rules:
-        -
-          :id: 9
-          :value: " ( $STerm | $ATerm ) $Close* ÷ ( $Close | $Sp | $Sep | $CR | $LF ) "  # override non-break in original with break