twitter_cldr 5.1.0 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0de7de286380d82bd1bb15f2153876905a48270c43c914f19a873d7a6314abe0
-  data.tar.gz: 100d3add1abf37b423dec0cacca0b899baed1e683abf8d1c43a7c8ce8303a3d0
+  metadata.gz: b616d55c343da67733837c0f1549329d895ea3758011b4c5dd8c90c3c631f53a
+  data.tar.gz: 3717867c2412adcc7a95ff1dfbe032e1754ade3cc005b4445d1f8ff644048b06
 SHA512:
-  metadata.gz: d5a310d97f73576229627ffa3f46eb8d1872b2dd3032e9ec9e7e54a502db896195b5a48119fb991a5647500eec9a222ceb8774e6f293e223313ad23ea25f238b
-  data.tar.gz: abaef5aa0122312d5aa7aa69eca068ef5f4ef270e145d168fd295cc4d7db1aaad77a1fde0e3bf31fa909d6c741d8d447a1b75588b9c297daafa26a053c9d244c
+  metadata.gz: f82323e912a622930f192a2ffe8b742ce1378feff847a082e04b7ea0feb5df215faa183861e31a35e69bc966c6222d0182689cdc5264de69263acbc60acce8ff
+  data.tar.gz: dc11c5d5e3ab6cc0f2cf3a6073686f7167ac6621252a0b6bf4c7f0b1eb53ae7134507aa0d66e5c0c57e7abc9b689a6438032b4982abe766b9c8550b6df0139b8

data/Rakefile

@@ -155,6 +155,11 @@ namespace :update do
     TwitterCldr::Resources::CollationTestsImporter.new.import
   end
 
+  desc 'Import text segmentation rules'
+  task :segment_rules do
+    TwitterCldr::Resources::SegmentRulesImporter.new.import
+  end
+
   desc 'Import (generate) bidi tests (should be executed using JRuby 1.7 in 1.9 mode)'
   task :bidi_tests do
     TwitterCldr::Resources::BidiTestImporter.new.import
@@ -195,11 +200,6 @@ namespace :update do
     TwitterCldr::Resources::TransformTestsImporter.new.import
   end
 
-  desc 'Import segment exceptions'
-  task :segment_exceptions do
-    TwitterCldr::Resources::Uli::SegmentExceptionsImporter.new.import
-  end
-
   desc 'Import segment tests'
   task :segment_tests do
     TwitterCldr::Resources::SegmentTestsImporter.new.import

data/lib/twitter_cldr.rb

@@ -59,6 +59,7 @@ module TwitterCldr
   def_delegator :resources, :resource_exists?
   def_delegator :resources, :locale_resource_exists?
   def_delegator :resources, :absolute_resource_path
+  def_delegator :resources, :resource_file_path
 
   class << self
 

data/lib/twitter_cldr/resources.rb

@@ -22,6 +22,7 @@ module TwitterCldr
     autoload :NumberFormatsImporter,          'twitter_cldr/resources/number_formats_importer'
     autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
     autoload :Properties,                     'twitter_cldr/resources/properties'
+    autoload :SegmentRulesImporter,           'twitter_cldr/resources/segment_rules_importer'
     autoload :RbnfTestImporter,               'twitter_cldr/resources/rbnf_test_importer'
     autoload :ReadmeRenderer,                 'twitter_cldr/resources/readme_renderer'
     autoload :RegexpAstGenerator,             'twitter_cldr/resources/regexp_ast_generator'
@@ -37,7 +38,6 @@ module TwitterCldr
     autoload :UnicodeDataImporter,            'twitter_cldr/resources/unicode_data_importer'
     autoload :UnicodeFileParser,              'twitter_cldr/resources/unicode_file_parser'
     autoload :UnicodePropertyAliasesImporter, 'twitter_cldr/resources/unicode_property_aliases_importer'
-    autoload :Uli,                            'twitter_cldr/resources/uli'
     autoload :ValidityDataImporter,           'twitter_cldr/resources/validity_data_importer'
 
     class << self
@@ -60,6 +60,7 @@ module TwitterCldr
           NumberFormatsImporter,
           PostalCodesImporter,
           RbnfTestImporter,
+          SegmentRulesImporter,
           SegmentTestsImporter,
           TailoringImporter,
           TerritoriesImporter,
@@ -74,12 +75,6 @@ module TwitterCldr
         ]
       end
 
-      def uli_importer_classes
-        @uli_importer_classes ||= [
-          Uli::SegmentExceptionsImporter
-        ]
-      end
-
       def property_importer_classes
         @property_importer_classes ||= [
           Properties::AgePropertyImporter,
@@ -107,7 +102,6 @@ module TwitterCldr
       def importer_classes
         @importer_classes ||=
           standard_importer_classes +
-          uli_importer_classes +
           property_importer_classes
       end
 

data/lib/twitter_cldr/resources/loader.rb

@@ -73,6 +73,12 @@ module TwitterCldr
         nil
       end
 
+      def resource_file_path(path)
+        file = File.join(*path.map(&:to_s))
+        file << '.yml' unless file.end_with?('.yml')
+        file
+      end
+
       private
 
       def locale_resource_path(locale, resource_name)
@@ -85,10 +91,6 @@ module TwitterCldr
         end
       end
 
-      def resource_file_path(path)
-        "#{File.join(*path.map(&:to_s))}.yml"
-      end
-
       def load_resource(path, merge_custom = true)
         base = YAML.load(read_resource_file(path))
         custom_path = File.join("custom", path)

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -37,7 +37,6 @@ module TwitterCldr
         currency_digits_and_rounding
         rbnf_root
         numbering_systems
-        segments_root
         territories_containment
         likely_subtags
         metazones

data/lib/twitter_cldr/resources/segment_rules_importer.rb

@@ -0,0 +1,202 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+require 'fileutils'
+require 'nokogiri'
+require 'yaml'
+
+module TwitterCldr
+  module Resources
+
+    class SegmentRulesImporter < Importer
+
+      # @TODO: moar boundary types
+      BOUNDARY_TYPES = {
+        'word'      => 'word',
+        'sentence'  => 'sentence',
+        'grapheme'  => 'grapheme',
+        'line'      => 'line'  # loose, normal, strict
+      }.freeze
+
+      TYPES_TO_ATTRS = {
+        'word'     => 'WordBreak',
+        'sentence' => 'SentenceBreak',
+        'grapheme' => 'GraphemeClusterBreak',
+        'line'     => 'LineBreak'
+      }.freeze
+
+      Locale = TwitterCldr::Shared::Locale
+
+      StateTable    = TwitterCldr::Segmentation::StateTable
+      StatusTable   = TwitterCldr::Segmentation::StatusTable
+      CategoryTable = TwitterCldr::Segmentation::CategoryTable
+
+      requirement :icu, Versions.icu_version
+      requirement :cldr, Versions.cldr_version
+      output_path File.join('shared', 'segments')
+      ruby_engine :jruby
+
+      def execute
+        each_locale do |locale, doc|
+          BOUNDARY_TYPES.each do |kind, icu_kind|
+            seg = doc.xpath(
+              "//ldml/segmentations/segmentation[@type=\"#{TYPES_TO_ATTRS[kind]}\"]"
+            )
+
+            rule_data = rule_data_for(icu_kind, locale, seg)
+
+            unless rule_data.empty?
+              output_dir = File.join(output_path, 'rules', locale)
+              output_file = File.join(output_dir, "#{kind}.yml")
+              FileUtils.mkdir_p(output_dir)
+              File.write(output_file, YAML.dump(rule_data))
+            end
+
+            suppressions = suppressions_for(icu_kind, locale, seg)
+
+            unless suppressions.empty?
+              output_dir = File.join(output_path, 'suppressions', locale)
+              output_file = File.join(output_dir, "#{kind}.yml")
+              FileUtils.mkdir_p(output_dir)
+              File.write(output_file, YAML.dump(suppressions))
+            end
+          end
+        end
+      end
+
+      private
+
+      def each_locale
+        return to_enum(__method__) unless block_given?
+
+        pattern = File.join(requirements[:cldr].common_path, 'segments', '*.xml')
+
+        Dir.glob(pattern).each do |file, ret|
+          locale = File.basename(file).chomp('.xml').tr('_', '-')
+          yield locale, Nokogiri::XML(File.read(file))
+        end
+      end
+
+      def rule_data_for(kind, locale, doc)
+        vars = doc.xpath('variables/variable')
+        rules = doc.xpath('segmentRules/rule')
+        result = {}
+
+        unless vars.empty? && rules.empty?
+          result.merge!(encode_rbbi_data(rbbi_data_for(kind, locale)))
+        end
+
+        result
+      end
+
+      def suppressions_for(kind, locale, doc)
+        suppressions = doc.xpath('suppressions/suppression').map(&:text)
+        return {} if suppressions.empty?
+
+        encode_suppressions(suppressions)
+      end
+
+      def encode_rbbi_data(data)
+        {
+          metadata: metadata_from(data.fHeader),
+          forward_table: StateTable.new(data.fFTable.fTable.to_a, data.fFTable.fFlags).dump16,
+          backward_table: StateTable.new(data.fRTable.fTable.to_a, data.fRTable.fFlags).dump16,
+          status_table: StatusTable.new(data.fStatusTable.to_a).dump,
+          category_table: encode_trie(data.fTrie),  # this really isn't a trie
+        }
+      end
+
+      def metadata_from(header)
+        { category_count: header.fCatCount }
+      end
+
+      def encode_suppressions(suppressions)
+        forwards_trie = TwitterCldr::Utils::Trie.new
+        backwards_trie = TwitterCldr::Utils::Trie.new
+
+        suppressions.each do |suppression|
+          forwards_trie.add(suppression.codepoints, true)
+          backwards_trie.add(suppression.reverse.codepoints, true)
+        end
+
+        {
+          forwards_trie: Marshal.dump(forwards_trie),
+          backwards_trie: Marshal.dump(backwards_trie)
+        }
+      end
+
+      def encode_trie(trie)
+        arr = [].tap do |results|
+          iter = trie.iterator
+
+          while iter.hasNext
+            range = iter.next
+            results << range_to_a(range)
+
+            # this should be the last entry, but for some reason ICU returns
+            # one more out-of-order range past the Unicode max
+            break if range.endCodePoint == 0x10FFFF
+          end
+        end
+
+        # @TODO: Distinguish between the 16- and 32-bit flavors
+        CategoryTable.new(arr).dump16.strip
+      end
+
+      def range_to_a(range)
+        [range.startCodePoint, range.endCodePoint, range.value]
+      end
+
+      def rbbi_data_for(kind, locale)
+        bundle = bundle_for(ulocale_class.new(locale))
+        brkf_name = bundle.getStringWithFallback("boundaries/#{kind}")
+        buffer = icu_binary.getData("#{brkiter_name}/#{brkf_name}")
+        rbbi_data_wrapper.get(buffer)
+      end
+
+      def bundle_for(locale)
+        @bundle ||= resource_bundle.getBundleInstance(brkiter_base_name, locale, locale_root)
+      end
+
+      def brkiter_name
+        @brkiter_name ||= icu_data.const_get(:ICU_BRKITR_NAME)
+      end
+
+      def brkiter_base_name
+        @brkiter_base_name ||= icu_data.const_get(:ICU_BRKITR_BASE_NAME)
+      end
+
+      def locale_root
+        @locale_root ||= resource_bundle.const_get(:OpenType).const_get(:LOCALE_ROOT)
+      end
+
+      def rbbi_data_wrapper
+        @rbbi_data_wrapper ||= requirements[:icu].get_class('com.ibm.icu.impl.RBBIDataWrapper')
+      end
+
+      def icu_binary
+        @icu_binary ||= requirements[:icu].get_class('com.ibm.icu.impl.ICUBinary')
+      end
+
+      def icu_data
+        @icu_data ||= requirements[:icu].get_class('com.ibm.icu.impl.ICUData')
+      end
+
+      def resource_bundle
+        @bundle_class ||= requirements[:icu].get_class('com.ibm.icu.impl.ICUResourceBundle')
+      end
+
+      def ulocale_class
+        @ulocale_class ||= requirements[:icu].get_class('com.ibm.icu.util.ULocale')
+      end
+
+      def output_path
+        params[:output_path]
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/segment_tests_importer.rb

@@ -11,7 +11,9 @@ module TwitterCldr
 
       TEST_FILES = [
         'ucd/auxiliary/WordBreakTest.txt',
-        'ucd/auxiliary/SentenceBreakTest.txt'
+        'ucd/auxiliary/SentenceBreakTest.txt',
+        'ucd/auxiliary/GraphemeBreakTest.txt',
+        'ucd/auxiliary/LineBreakTest.txt'
       ]
 
       requirement :unicode, Versions.unicode_version, TEST_FILES

data/lib/twitter_cldr/segmentation.rb

@@ -5,13 +5,15 @@
 
 module TwitterCldr
   module Segmentation
-    autoload :BreakIterator,  'twitter_cldr/segmentation/break_iterator'
-    autoload :BreakRule,      'twitter_cldr/segmentation/rule'
-    autoload :Cursor,         'twitter_cldr/segmentation/cursor'
-    autoload :NoBreakRule,    'twitter_cldr/segmentation/rule'
-    autoload :Parser,         'twitter_cldr/segmentation/parser'
-    autoload :Rule,           'twitter_cldr/segmentation/rule'
-    autoload :RuleSet,        'twitter_cldr/segmentation/rule_set'
-    autoload :RuleSetBuilder, 'twitter_cldr/segmentation/rule_set_builder'
+    autoload :BreakIterator,    'twitter_cldr/segmentation/break_iterator'
+    autoload :CategoryTable,    'twitter_cldr/segmentation/category_table'
+    autoload :Cursor,           'twitter_cldr/segmentation/cursor'
+    autoload :Metadata,         'twitter_cldr/segmentation/metadata'
+    autoload :NullSuppressions, 'twitter_cldr/segmentation/null_suppressions'
+    autoload :RuleSet,          'twitter_cldr/segmentation/rule_set'
+    autoload :StateMachine,     'twitter_cldr/segmentation/state_machine'
+    autoload :StateTable,       'twitter_cldr/segmentation/state_table'
+    autoload :StatusTable,      'twitter_cldr/segmentation/status_table'
+    autoload :Suppressions,     'twitter_cldr/segmentation/suppressions'
   end
 end

data/lib/twitter_cldr/segmentation/break_iterator.rb

@@ -25,29 +25,33 @@ module TwitterCldr
       end
 
       def each_grapheme_cluster(str, &block)
-        raise NotImplementedError,
-          "Grapheme segmentation is not currently supported."
+        rule_set = rule_set_for('grapheme')
+        each_boundary(rule_set, str, &block)
       end
 
       def each_line(str, &block)
-        raise NotImplementedError,
-          "Line segmentation is not currently supported."
+        rule_set = rule_set_for('line')
+        each_boundary(rule_set, str, &block)
       end
 
       private
 
       def each_boundary(rule_set, str)
-        if block_given?
-          rule_set.each_boundary(str).each_cons(2) do |start, stop|
-            yield str[start...stop], start, stop
-          end
-        else
-          to_enum(__method__, rule_set, str)
+        return to_enum(__method__, rule_set, str) unless block_given?
+
+        rule_set.each_boundary(str).each_cons(2) do |start, stop|
+          yield str[start...stop], start, stop
         end
       end
 
       def rule_set_for(boundary_type)
-        RuleSet.load(locale, boundary_type, options)
+        rule_set_cache[boundary_type] ||= RuleSet.create(
+          locale, boundary_type, options
+        )
+      end
+
+      def rule_set_cache
+        @rule_set_cache ||= {}
       end
     end
   end

data/lib/twitter_cldr/segmentation/category_table.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+
+module TwitterCldr
+  module Segmentation
+    class CategoryTable
+      PACK_FMT_16 = 'NNn'.freeze
+
+      class << self
+        def load16(data)
+          data = Base64.decode64(data)
+
+          new(
+            (0...data.size).step(10).map do |i|
+              data[i...(i + 10)].unpack(PACK_FMT_16)
+            end
+          )
+        end
+      end
+
+      attr_reader :values
+
+      def initialize(values)
+        @values = values
+      end
+
+      def get(codepoint)
+        find(codepoint)[2]
+      end
+
+      def dump16
+        data = ''.b.tap do |result|
+          values.each do |vals|
+            result << vals.pack(PACK_FMT_16)
+          end
+        end
+
+        Base64.encode64(data)
+      end
+
+      private
+
+      def find(codepoint)
+        values.bsearch do |entry|
+          next -1 if codepoint < entry[0]
+          next 1 if codepoint > entry[1]
+          0
+        end
+      end
+    end
+  end
+end