twitter_cldr 1.6.1 → 1.6.2

data/History.txt

@@ -1,3 +1,7 @@
+=== 1.6.2
+
+* Collation tries now loaded from marshal dumps, collation running time improved by ~80%.
+
 === 1.6.1
 
 * Added case-first collation element tailoring support for languages like Danish.

data/Rakefile

@@ -11,6 +11,8 @@ require 'digest'
 require 'rspec/core/rake_task'
 require 'rubygems/package_task'
 
+require './lib/twitter_cldr'
+
 Bundler::GemHelper.install_tasks
 
 task :default => :spec
@@ -42,9 +44,7 @@ namespace :resources do
   namespace :update do
     desc 'Import tailoring resources from CLDR data (should be executed using JRuby 1.7 in 1.9 mode)'
     task :tailoring do
-      require './lib/twitter_cldr'
-
-      importer = TwitterCldr::Resources::Import::Tailoring.new(
+      importer = TwitterCldr::Resources::TailoringImporter.new(
           ENV.fetch('CLDR_DATA_PATH', '../cldr-tailoring/'),
           './resources/collation/tailoring',
           ENV.fetch('ICU4J_JAR_PATH', '../icu4j-49_1.jar')
@@ -52,6 +52,11 @@ namespace :resources do
 
       TwitterCldr.supported_locales.each { |locale| importer.import(locale) }
     end
+
+    desc 'Updates default and tailoring tries dumps'
+    task :tries do
+      TwitterCldr::Resources::TriesDumper.update_dumps
+    end
   end
 end
 

data/lib/twitter_cldr/collation/collator.rb

@@ -11,8 +11,6 @@ module TwitterCldr
     #
     class Collator
 
-      FRACTIONAL_UCA_SHORT_RESOURCE = 'collation/FractionalUCA_SHORT.txt'
-
       attr_accessor :locale
 
       def initialize(locale = nil)
@@ -53,7 +51,7 @@ module TwitterCldr
       end
 
       def load_trie
-        @locale ? self.class.tailored_fce_trie(@locale) : self.class.default_fce_trie
+        @locale ? self.class.tailored_trie(@locale) : self.class.default_trie
       end
 
       def get_integer_code_points(code_points)
@@ -138,24 +136,24 @@ module TwitterCldr
 
       class << self
 
-        # Loads and memoizes the default Fractional Collation Elements trie.
+        # Loads and memoizes the default fractional collation elements trie.
         #
-        def default_fce_trie
-          @default_fce_trie ||= TwitterCldr::Collation::TrieBuilder.load_trie(FRACTIONAL_UCA_SHORT_RESOURCE).lock
+        def default_trie
+          @default_trie ||= TwitterCldr::Collation::TrieLoader.load_default_trie.lock
         end
 
-        def tailored_fce_trie(locale)
-          tailored_fce_tries_cache[locale]
+        def tailored_trie(locale)
+          tailored_tries_cache[locale]
         end
 
         private
 
-        def tailored_fce_tries_cache
-          @tailored_fce_tries_cache ||= Hash.new { |hash, locale| hash[locale] = load_tailored_trie(locale) }
+        def tailored_tries_cache
+          @tailored_tries_cache ||= Hash.new { |hash, locale| hash[locale] = load_tailored_trie(locale) }
         end
 
         def load_tailored_trie(locale)
-          TwitterCldr::Collation::TrieBuilder.load_tailored_trie(locale, default_fce_trie).lock
+          TwitterCldr::Collation::TrieLoader.load_tailored_trie(locale, default_trie).lock
         end
 
       end

data/lib/twitter_cldr/collation/trie.rb

@@ -91,6 +91,14 @@ module TwitterCldr
         [last_with_value.value, last_prefix_size, last_with_value.to_trie]
       end
 
+      def marshal_dump
+        @root
+      end
+
+      def marshal_load(root)
+        @root = root
+      end
+
       def to_hash
         @root.subtrie_hash
       end

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -10,35 +10,32 @@ module TwitterCldr
     #
     module TrieBuilder
 
-      # Fractional collation element regexp
-      FCE_REGEXP = /^((?:[0-9A-F]+)(?:\s[0-9A-F]+)*);\s((?:\[.*?\])(?:\[.*?\])*)/
+      COLLATION_ELEMENT_REGEXP = /^((?:[0-9A-F]+)(?:\s[0-9A-F]+)*);\s((?:\[.*?\])(?:\[.*?\])*)/
+
+      FRACTIONAL_UCA_SHORT_PATH = File.join(TwitterCldr::RESOURCES_DIR, 'collation', 'FractionalUCA_SHORT.txt')
 
       class << self
 
-        def load_trie(resource)
-          parse_trie(load_resource(resource))
+        def load_default_trie
+          File.open(FRACTIONAL_UCA_SHORT_PATH, 'r') { |table| parse_collation_elements_table(table) }
         end
 
         def load_tailored_trie(locale, fallback)
           build_tailored_trie(tailoring_data(locale), fallback)
         end
 
-        def parse_trie(table, trie = TwitterCldr::Collation::Trie.new)
-          table.lines.each do |line|
-            trie.set(parse_code_points($1), parse_collation_element($2)) if FCE_REGEXP =~ line
-          end
-
-          trie
-        end
-
         def tailoring_data(locale)
           TwitterCldr.get_resource(:collation, :tailoring, locale)
         end
 
         private
 
-        def load_resource(resource)
-          open(File.join(TwitterCldr::RESOURCES_DIR, resource), 'r')
+        def parse_collation_elements_table(table, trie = TwitterCldr::Collation::Trie.new)
+          table.lines.each do |line|
+            trie.set(parse_code_points($1), parse_collation_element($2)) if COLLATION_ELEMENT_REGEXP =~ line
+          end
+
+          trie
         end
 
         def parse_code_points(string)
@@ -54,7 +51,7 @@ module TwitterCldr
         def build_tailored_trie(tailoring_data, fallback)
           trie = TwitterCldr::Collation::TrieWithFallback.new(fallback)
 
-          parse_trie(tailoring_data[:tailored_table], trie)
+          parse_collation_elements_table(tailoring_data[:tailored_table], trie)
           copy_expansions(trie, fallback, parse_suppressed_starters(tailoring_data[:suppressed_contractions]))
 
           trie

data/lib/twitter_cldr/collation/trie_loader.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Collation
+
+    # Builds a fractional collation elements Trie from the file containing a fractional collation elements table.
+    #
+    module TrieLoader
+
+      DUMPS_DIR = File.join(TwitterCldr::RESOURCES_DIR, 'collation', 'tries')
+
+      DEFAULT_TRIE_LOCALE = :default
+
+      class << self
+
+        def load_default_trie
+          load_trie
+        end
+
+        def load_tailored_trie(locale, fallback)
+          trie = load_trie(locale)
+          trie.fallback = fallback
+          trie
+        end
+
+        def dump_path(locale)
+          File.join(DUMPS_DIR, "#{locale}.dump")
+        end
+
+        private
+
+        def load_trie(locale = DEFAULT_TRIE_LOCALE)
+          load_dump(locale) do |dump|
+            Marshal.load(dump)
+          end
+        end
+
+        def load_dump(locale, &block)
+          File.open(dump_path(locale), 'r', &block)
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/collation/trie_with_fallback.rb

@@ -26,17 +26,19 @@ module TwitterCldr
     #
     # This special behavior of the #find_prefix method allows 'hiding' fallback keys that contain more than one element
     # by adding their one element prefixes to the trie itself. This feature is useful for some applications, e.g., for
-    # suppressing contractions in a tailored FCE trie.
+    # suppressing contractions in a tailored fractional collation elements trie.
     #
     class TrieWithFallback < TwitterCldr::Collation::Trie
 
+      attr_accessor :fallback
+
       def initialize(fallback)
         super()
-        @fallback = fallback
+        self.fallback = fallback
       end
 
       def get(key)
-        super || @fallback.get(key)
+        super || fallback.get(key)
       end
 
       def find_prefix(key)
@@ -45,7 +47,7 @@ module TwitterCldr
         if prefix_size > 0
           [value, prefix_size, suffixes]
         else
-          @fallback.find_prefix(key)
+          fallback.find_prefix(key)
         end
       end
 

data/lib/twitter_cldr/collation.rb

@@ -10,6 +10,7 @@ module TwitterCldr
     autoload :SortKeyBuilder,            'twitter_cldr/collation/sort_key_builder'
     autoload :Trie,                      'twitter_cldr/collation/trie'
     autoload :TrieBuilder,               'twitter_cldr/collation/trie_builder'
+    autoload :TrieLoader,                'twitter_cldr/collation/trie_loader'
     autoload :TrieWithFallback,          'twitter_cldr/collation/trie_with_fallback'
   end
 end

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -0,0 +1,203 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+require 'yaml'
+require 'java'
+
+module TwitterCldr
+  module Resources
+    # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version 49.1 (available at
+    # http://download.icu-project.org/files/icu4j/49.1/icu4j-49_1.jar).
+    #
+    class TailoringImporter
+
+      SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
+      SIMPLE_RULES      = %w[p s t i]
+      LEVEL_RULE_REGEXP = /^(p|s|t|i)(c?)$/
+
+      IGNORED_TAGS = %w[reset text #comment]
+
+      LAST_BYTE_MASK = 0xFF
+
+      LOCALES_MAP = {
+          :'zh-Hant' => :'zh_Hant',
+          :id => :root,
+          :it => :root,
+          :ms => :root,
+          :nl => :root,
+          :pt => :root
+      }
+
+      EMPTY_TAILORING_DATA = { 'collator_options' => {}, 'tailored_table' => '', 'suppressed_contractions' => '' }
+
+      class ImportError < RuntimeError; end
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing CLDR tailoring data (available at
+      #                 http://unicode.org/cldr/trac/browser/tags/release-21/common/collation/
+      #                 or as a part of CLDR release at http://cldr.unicode.org/index/downloads)
+      #
+      #   output_path - output directory for imported YAML files
+      #
+      #   icu4j_path  - path to ICU4J jar file
+      #
+      def initialize(input_path, output_path, icu4j_path)
+        require icu4j_path
+
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import(locale)
+        print "Importing %8s\t--\t" % locale
+
+        if tailoring_present?(locale)
+          dump(locale, tailoring_data(locale))
+          puts "Done."
+        else
+          dump(locale, EMPTY_TAILORING_DATA)
+          puts "Missing (generated empty tailoring resource)."
+        end
+      rescue ImportError => e
+        puts "Error: #{e.message}"
+      end
+
+      private
+
+      def dump(locale, data)
+        File.open(resource_file_path(locale), 'w') { |file| YAML.dump(data, file) }
+      end
+
+      def tailoring_present?(locale)
+        File.file?(locale_file_path(locale))
+      end
+
+      def translated_locale(locale)
+        LOCALES_MAP.fetch(locale, locale)
+      end
+
+      def locale_file_path(locale)
+        File.join(@input_path, "#{translated_locale(locale)}.xml")
+      end
+
+      def resource_file_path(locale)
+        File.join(@output_path, "#{locale}.yml")
+      end
+
+      def tailoring_data(locale)
+        doc = File.open(locale_file_path(locale)) { |file| Nokogiri::XML(file) }
+        collations = doc.at_xpath('//collations')
+
+        collation_alias = collations.at_xpath('alias[@path="//ldml/collations"]')
+        aliased_locale = collation_alias && collation_alias.attr('source')
+
+        return tailoring_data(aliased_locale) if aliased_locale
+
+        standard_tailoring = collations.at_xpath('collation[@type="standard"]')
+
+        {
+            'collator_options'        => parse_collator_options(standard_tailoring),
+            'tailored_table'          => parse_tailorings(standard_tailoring, locale),
+            'suppressed_contractions' => parse_suppressed_contractions(standard_tailoring)
+        }
+      end
+
+      def parse_tailorings(data, locale)
+        rules = data && data.at_xpath('rules')
+
+        return '' unless rules
+
+        collator = Java::ComIbmIcuText::Collator.get_instance(Java::JavaUtil::Locale.new(locale.to_s))
+
+        rules.children.map do |child|
+          validate_tailoring_rule(child)
+
+          if child.name =~ LEVEL_RULE_REGEXP
+            if $2.empty?
+              table_entry_for_rule(collator, child.text)
+            else
+              child.text.chars.map { |char| table_entry_for_rule(collator, char) }
+            end
+          elsif child.name == 'x'
+            context = ''
+            child.children.each_with_object([]) do |c, memo|
+              if SIMPLE_RULES.include?(c.name)
+                memo << table_entry_for_rule(collator, context + c.text)
+              elsif c.name == 'context'
+                context = c.text
+              elsif c.name != 'extend'
+                raise ImportError, "Rule '#{c.name}' inside <x></x> is not supported."
+              end
+            end
+          else
+            raise ImportError, "Tag '#{child.name}' is not supported." unless IGNORED_TAGS.include?(child.name)
+          end
+        end.flatten.compact.join("\n")
+      end
+
+      def table_entry_for_rule(collator, tailored_value)
+        code_points = get_code_points(tailored_value)
+
+        collation_elements = get_collation_elements(collator, tailored_value).map do |ce|
+          ce.map { |l| l.to_s(16).upcase }.join(', ')
+        end
+
+        "#{code_points.join(' ')}; [#{collation_elements.join('][')}]"
+      end
+
+      def parse_suppressed_contractions(data)
+        node = data && data.at_xpath('suppress_contractions')
+        node ? Java::ComIbmIcuText::UnicodeSet.to_array(Java::ComIbmIcuText::UnicodeSet.new(node.text)).to_a.join : ''
+      end
+
+      def parse_collator_options(data)
+        options = {}
+
+        if data
+          case_first_setting = data.at_xpath('settings[@caseFirst]')
+          options['case_first'] = case_first_setting.attr('caseFirst').to_sym if case_first_setting
+        end
+
+        options
+      end
+
+      def validate_tailoring_rule(rule)
+        return if IGNORED_TAGS.include?(rule.name)
+
+        raise ImportError, "Rule '#{rule.name}' is not supported." unless SUPPORTED_RULES.include?(rule.name)
+      end
+
+      def get_collation_elements(collator, string)
+        iter = collator.get_collation_element_iterator(string)
+
+        collation_elements = []
+        ce = iter.next
+
+        while ce != Java::ComIbmIcuText::CollationElementIterator::NULLORDER
+          p1 = (ce >> 24) & LAST_BYTE_MASK
+          p2 = (ce >> 16) & LAST_BYTE_MASK
+
+          primary   = p2.zero? ? p1 : (p1 << 8) + p2
+          secondary = (ce >> 8) & LAST_BYTE_MASK
+          tertiarly = ce & LAST_BYTE_MASK
+
+          collation_elements << [primary, secondary, tertiarly]
+
+          ce = iter.next
+        end
+
+        collation_elements
+      end
+
+      def get_code_points(string)
+        TwitterCldr::Normalization::NFD.normalize_code_points(TwitterCldr::Utils::CodePoints.from_string(string))
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/tries_dumper.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+
+    module TriesDumper
+
+      class << self
+
+        def update_dumps
+          update_default_trie_dump
+          TwitterCldr.supported_locales.each { |locale| update_tailoring_trie_dump(locale) }
+        end
+
+        private
+
+        def update_default_trie_dump
+          save_trie_dump(TwitterCldr::Collation::TrieLoader::DEFAULT_TRIE_LOCALE, default_trie)
+        end
+
+        def update_tailoring_trie_dump(locale)
+          save_trie_dump(locale, TwitterCldr::Collation::TrieBuilder.load_tailored_trie(locale, @default_trie))
+        end
+
+        def save_trie_dump(locale, trie)
+          path = TwitterCldr::Collation::TrieLoader.dump_path(locale)
+          FileUtils.mkdir_p(File.dirname(path))
+
+          File.open(path, 'w') { |f| f.write(Marshal.dump(trie)) }
+        end
+
+        def default_trie
+          @default_trie ||= TwitterCldr::Collation::TrieBuilder.load_default_trie
+        end
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources.rb

@@ -5,7 +5,8 @@
 
 module TwitterCldr
   module Resources
-    autoload :Loader, 'twitter_cldr/resources/loader'
-    autoload :Import, 'twitter_cldr/resources/import'
+    autoload :Loader,            'twitter_cldr/resources/loader'
+    autoload :TailoringImporter, 'twitter_cldr/resources/tailoring_importer'
+    autoload :TriesDumper,       'twitter_cldr/resources/tries_dumper'
   end
 end

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "1.6.1"
+  VERSION = "1.6.2"
 end

data/spec/collation/collation_spec.rb

@@ -31,7 +31,7 @@ describe 'Unicode Collation Algorithm' do
 
     previous_sort_key = previous_code_points = previous_hex_code_points = nil
 
-    open(file_path, 'r:utf-8') do |file|
+    File.open(file_path, 'r:utf-8') do |file|
       file.each do |line|
         next unless /^([0-9A-F ]+);/ =~ line
 
@@ -79,7 +79,9 @@ END
 
     print 'extracting CollationTest_CLDR_NON_IGNORABLE.txt ... '
     Zip::ZipFile.open(zip_file.path) do |zip|
-      open(FULL_COLLATION_TEST_PATH, 'w') { |file| file.write(zip.read('CollationAuxiliary/CollationTest_CLDR_NON_IGNORABLE.txt')) }
+      File.open(FULL_COLLATION_TEST_PATH, 'w') do |file|
+        file.write(zip.read('CollationAuxiliary/CollationTest_CLDR_NON_IGNORABLE.txt'))
+      end
     end
     zip_file.unlink