twitter_cldr 1.5.0 → 1.6.0

data/spec/collation/collation_spec.rb

@@ -38,7 +38,7 @@ describe 'Unicode Collation Algorithm' do
         current_code_points = $1.split
         current_hex_code_points = current_code_points.map { |cp| cp.to_i(16) }
 
-        current_sort_key = collator.sort_key(current_code_points)
+        current_sort_key = collator.get_sort_key(current_code_points)
 
         if previous_sort_key
           result = (previous_sort_key <=> current_sort_key).nonzero? || (previous_hex_code_points <=> current_hex_code_points)

data/spec/collation/collator_spec.rb

@@ -9,58 +9,108 @@ include TwitterCldr::Collation
 
 describe Collator do
 
-  before :each do
-    Collator.instance_variable_set(:@trie, nil)
-  end
+  let(:trie) { Trie.new }
+
+  before(:each) { clear_fce_tries_cache }
+  after(:all)   { clear_fce_tries_cache }
+
+  describe '.default_fce_trie' do
+    before(:each) do
+      clear_default_fce_trie_cache
+      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { trie }
+    end
+
+    it 'returns default fractional collation elements trie' do
+      Collator.default_fce_trie.should == trie
+    end
+
+    it 'loads the trie only once' do
+      Collator.default_fce_trie.object_id.should == Collator.default_fce_trie.object_id
+    end
 
-  after :all do
-    Collator.instance_variable_set(:@trie, nil)
+    it 'locks the trie' do
+      Collator.default_fce_trie.should be_locked
+    end
   end
 
-  describe '.trie' do
-    it 'returns collation elements trie' do
-      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { 'trie' }
-      Collator.trie.should == 'trie'
+  describe '.tailored_fce_trie' do
+    let(:locale) { :ru }
+
+    before(:each) do
+      clear_tailored_fce_tries_cache
+      stub(Collator).default_fce_trie { trie }
+      mock(TrieBuilder).load_tailored_trie(locale, Collator.default_fce_trie) { trie }
+    end
+
+    it 'returns default fractional collation elements trie' do
+      Collator.tailored_fce_trie(locale).should == trie
     end
 
     it 'loads the trie only once' do
-      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { 'trie' }
+      Collator.tailored_fce_trie(locale).object_id.should == Collator.tailored_fce_trie(locale).object_id
+    end
 
-      Collator.trie.object_id.should == Collator.trie.object_id
+    it 'locks the trie' do
+      Collator.tailored_fce_trie(locale).should be_locked
     end
   end
 
-  describe '#trie' do
-    it 'delegates to the class method' do
-      mock(Collator).trie { 'trie' }
-      Collator.new.trie.should == 'trie'
+  describe '#initialize' do
+    before(:each) { stub(TrieBuilder).load_trie { trie } }
+    before(:each) { any_instance_of(Collator) { |c| stub(c).load_trie { trie } } }
+
+    it 'initializes default collator if locale is not specified' do
+      Collator.new.locale.should be_nil
     end
 
-    it 'calls class method only once' do
-      mock(Collator).trie { 'trie' }
+    it 'initialized tailored collator if locale is provided' do
+      Collator.new(:ru).locale.should == :ru
+    end
 
-      collator = Collator.new
-      collator.trie.object_id.should == collator.trie.object_id
+    it 'converts locale' do
+      Collator.new(:no).locale.should == :nb
     end
   end
 
-  describe '#sort_key' do
-    let(:collator)        { Collator.new }
-    let(:string)          { 'abc' }
-    let(:code_points_hex) { %w[0061 0062 0063] }
-    let(:code_points)     { code_points_hex.map { |cp| cp.to_i(16) } }
-    let(:sort_key)        { [9986, 10498, 11010, 0, 1282, 1282, 1282, 0, 1282, 1282, 1282] }
+  describe '#get_collation_elements' do
+    let(:collator)           { Collator.new }
+    let(:string)             { 'abc' }
+    let(:code_points_hex)    { %w[0061 0062 0063] }
+    let(:code_points)        { code_points_hex.map { |cp| cp.to_i(16) } }
+    let(:collation_elements) { [[39, 5, 5], [41, 5, 5], [43, 5, 5]] }
 
-    before(:each) { mock(collator).sort_key_for_code_points(code_points) { sort_key } }
+    before :each do
+      mock(TwitterCldr::Normalization::NFD).normalize_code_points(code_points_hex) { code_points_hex }
+      stub(TwitterCldr::Normalization::Base).combining_class_for { 0 }
+    end
+
+    it 'returns collation elements for a string' do
+      collator.get_collation_elements(string).should == collation_elements
+    end
+
+    it 'returns collation elements for an array of code points (represented as hex strings)' do
+      collator.get_collation_elements(code_points_hex).should == collation_elements
+    end
+  end
+
+  describe '#get_sort_key' do
+    let(:collator)           { Collator.new }
+    let(:string)             { 'abc' }
+    let(:code_points_hex)    { %w[0061 0062 0063] }
+    let(:collation_elements) { [[39, 5, 5], [41, 5, 5], [43, 5, 5]] }
+    let(:sort_key)           { [39, 41, 43, 1, 7, 1, 7] }
+
+    before(:each) { stub(TrieBuilder).load_trie { trie } }
+    before(:each) { mock(TwitterCldr::Collation::SortKeyBuilder).build(collation_elements) { sort_key } }
 
     it 'calculates sort key for a string' do
-      mock(TwitterCldr::Utils::CodePoints).from_string(string) { code_points_hex }
-      collator.sort_key(string).should == sort_key
+      mock(collator).get_collation_elements(string) { collation_elements }
+      collator.get_sort_key(string).should == sort_key
     end
 
     it 'calculates sort key for an array of code points (represented as hex strings)' do
-      dont_allow(TwitterCldr::Utils::CodePoints).from_string(string)
-      collator.sort_key(code_points_hex).should == sort_key
+      mock(collator).get_collation_elements(code_points_hex) { collation_elements }
+      collator.get_sort_key(code_points_hex).should == sort_key
     end
   end
 
@@ -69,6 +119,8 @@ describe Collator do
     let(:sort_key)         { [1, 3, 8, 9] }
     let(:another_sort_key) { [6, 8, 9, 2] }
 
+    before(:each) { stub(Collator).default_fce_trie { trie } }
+
     it 'compares strings by sort keys' do
       stub_sort_key(collator, 'foo', sort_key)
       stub_sort_key(collator, 'bar', another_sort_key)
@@ -77,41 +129,61 @@ describe Collator do
       collator.compare('bar', 'foo').should == 1
     end
 
-    it 'returns 0 without computing sort keys if strings are equal' do
-      dont_allow(collator).sort_key
+    it 'returns 0 without computing sort keys if the strings are equal' do
+      dont_allow(collator).get_sort_key
 
       collator.compare('foo', 'foo').should == 0
     end
+  end
 
-    it 'compares strings by code points if the sort keys are equal' do
-      stub(collator).sort_key { sort_key }
+  describe 'sorting' do
+    let(:collator)  { Collator.new }
+    let(:sort_keys) { [['aaa', [1, 2, 3]], ['abc', [1, 3, 4]], ['bca', [2, 5, 9]]] }
+    let(:array)     { %w[bca aaa abc] }
+    let(:sorted)    { %w[aaa abc bca] }
 
-      collator.compare('bar', 'foo').should == -1
+    before :each do
+      stub(Collator).default_fce_trie { trie }
+      sort_keys.each { |s, key| mock_sort_key(collator, s, key) }
     end
-  end
 
-  describe '#sort' do
-    let(:collator) { Collator.new }
+    describe '#sort' do
+      it 'sorts strings by sort keys' do
+        collator.sort(array).should == sorted
+      end
 
-    it 'sorts strings by sort keys' do
-      [['aaa', [1, 2, 3]], ['abc', [1, 3, 4]], ['bca', [2, 5, 9]]].each { |s, key| mock_sort_key(collator, s, key) }
-
-      collator.sort(%w[bca aaa abc]).should == %w[aaa abc bca]
+      it 'does not change the original array' do
+        lambda { collator.sort(array) }.should_not change { array }
+      end
     end
 
-    it 'sorts strings with equal sort keys by code points' do
-      [['aaa', [1, 2, 3]], ['abc', [1, 2, 3]], ['bca', [1, 2, 3]]].each { |s, key| mock_sort_key(collator, s, key) }
-
-      collator.sort(%w[bca abc aaa]).should == %w[aaa abc bca]
+    describe '#sort!' do
+      it 'sorts strings array by sort keys in-place ' do
+        collator.sort!(array)
+        array.should == sorted
+      end
     end
   end
 
   def mock_sort_key(collator, string, sort_key)
-    mock(collator).sort_key(TwitterCldr::Utils::CodePoints.from_string(string)) { sort_key }
+    mock(collator).get_sort_key(string) { sort_key }
   end
 
   def stub_sort_key(collator, string, sort_key)
-    stub(collator).sort_key(TwitterCldr::Utils::CodePoints.from_string(string)) { sort_key }
+    stub(collator).get_sort_key(string) { sort_key }
+  end
+
+  def clear_fce_tries_cache
+    clear_default_fce_trie_cache
+    clear_tailored_fce_tries_cache
+  end
+
+  def clear_default_fce_trie_cache
+    Collator.instance_variable_set(:@default_fce_trie, nil)
+  end
+
+  def clear_tailored_fce_tries_cache
+    Collator.instance_variable_set(:@tailored_fce_tries_cache, nil)
   end
 
 end

data/spec/collation/sort_key_builder_spec.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe SortKeyBuilder do
+
+  let(:sort_key) { SortKeyBuilder.new(collation_elements) }
+  let(:collation_elements) { [[63, 13, 149], [66, 81, 143]] }
+  let(:sort_key_bytes) { [63, 66, 1, 13, 81, 1, 149, 143] }
+
+  describe '.build' do
+    it 'returns a sort key for a given array of collation elements' do
+      sort_key = SortKeyBuilder.new(collation_elements)
+
+      mock(SortKeyBuilder).new(collation_elements) { sort_key }
+      mock(sort_key).bytes_array { sort_key_bytes }
+
+      SortKeyBuilder.build(collation_elements).should == sort_key_bytes
+    end
+  end
+
+  describe '#initialize' do
+    it 'assigns collation elements array' do
+      SortKeyBuilder.new(collation_elements).collation_elements.should == collation_elements
+    end
+  end
+
+  describe '#bytes_array' do
+    it 'builds sort key bytes' do
+      sort_key.bytes_array.should == sort_key_bytes
+    end
+
+    it 'builds bytes array only once' do
+      mock(sort_key).build_bytes_array { sort_key_bytes }
+      sort_key.bytes_array.object_id == sort_key.bytes_array.object_id
+    end
+
+    it 'compresses primary weights' do
+      SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x908, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
+                          [0x7A, 0x72,           0x73,           0x75, 0x3,    0x9, 0x08,     0x7A, 0x73, 1, 1]
+
+      SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x9508, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
+                          [0x7A, 0x72,           0x73,           0x75, 0xFF,   0x95, 0x08,     0x7A, 0x73, 1, 1]
+    end
+
+    it 'works when there is an ignorable primary weight in the middle' do
+      SortKeyBuilder.new([[0x1312, 0, 0], [0, 0, 0], [0x1415, 0, 0]]).bytes_array.should == [0x13, 0x12, 0x14, 0x15, 1, 1]
+    end
+
+    it 'do not compress single byte primary weights' do
+      SortKeyBuilder.new([[0x13, 0, 0], [0x13, 0, 0]]).bytes_array.should == [0x13, 0x13, 1, 1]
+    end
+
+    it 'resets primary lead bytes counter after a single byte weight' do
+      SortKeyBuilder.new([[0x1415, 0, 0], [0x13, 0, 0], [0x13, 0, 0], [0x1412, 0, 0]]).bytes_array.should == [0x14, 0x15, 0x13, 0x13, 0x14, 0x12, 1, 1]
+    end
+
+    it 'compresses only allowed primary weights' do
+      SortKeyBuilder.new([[0x812, 0, 0], [0x811, 0, 0]]).bytes_array.should == [0x8, 0x12, 0x8, 0x11, 1, 1]
+    end
+
+    it 'compresses secondary weights' do
+      SortKeyBuilder.new([[0, 5, 0], [0, 5, 0], [0, 141, 0], [0, 5, 0], [0, 5, 0]]).bytes_array.should == [1, 133, 141, 6, 1]
+    end
+
+    it 'compresses tertiary weights' do
+      SortKeyBuilder.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]]).bytes_array.should == [1, 1, 132, 167, 6]
+    end
+
+    it 'compresses secondary and tertiary weights into multiple bytes if necessary' do
+      SortKeyBuilder.new([[0, 5, 5]] * 100).bytes_array.should == [1, 69, 40, 1, 48, 48, 18]
+    end
+  end
+
+end

data/spec/collation/tailoring_spec.rb

@@ -0,0 +1,137 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe 'Unicode collation tailoring' do
+
+  describe 'tailoring support' do
+    before(:each) do
+      stub(Collator).default_fce_trie { TrieBuilder.parse_trie(fractional_uca_short_stub) }
+      stub(TwitterCldr::Normalization::NFD).normalize_code_points { |code_points| code_points }
+      stub(TwitterCldr).get_resource(:collation, :tailoring, locale) { YAML.load(tailoring_resource_stub) }
+    end
+
+    let(:locale)            { :some_locale }
+    let(:default_collator)  { Collator.new }
+    let(:tailored_collator) { Collator.new(locale) }
+
+    describe 'tailoring rules support' do
+      it 'tailored collation elements are used' do
+        default_collator.get_collation_elements(%w[0490]).should  == [[0x5C1A, 5, 0x93], [0, 0xDBB9, 9]]
+        tailored_collator.get_collation_elements(%w[0490]).should == [[0x5C1B, 5, 0x86]]
+
+        default_collator.get_collation_elements(%w[0491]).should  == [[0x5C1A, 5, 9], [0, 0xDBB9, 9]]
+        tailored_collator.get_collation_elements(%w[0491]).should == [[0x5C1B, 5, 5]]
+      end
+
+      it 'original contractions for tailored elements are applied' do
+        default_collator.get_collation_elements(%w[0491 0306]).should  == [[0x5C, 0xDB, 9]]
+        tailored_collator.get_collation_elements(%w[0491 0306]).should == [[0x5C, 0xDB, 9]]
+      end
+    end
+
+    describe 'contractions suppressing support' do
+      it 'suppressed contractions are ignored' do
+        default_collator.get_collation_elements(%w[041A 0301]).should  == [[0x5CCC, 5, 0x8F]]
+        tailored_collator.get_collation_elements(%w[041A 0301]).should == [[0x5C6C, 5, 0x8F], [0, 0x8D, 5]]
+      end
+
+      it 'non-suppressed contractions are used' do
+        default_collator.get_collation_elements(%w[0415 0306]).should  == [[0x5C36, 5, 0x8F]]
+        tailored_collator.get_collation_elements(%w[0415 0306]).should == [[0x5C36, 5, 0x8F]]
+      end
+    end
+
+    let(:fractional_uca_short_stub) do
+      <<END
+# collation elements from default FCE table
+0301; [, 8D, 05]
+0306; [, 91, 05]
+041A; [5C 6C, 05, 8F] # К
+0413; [5C 1A, 05, 8F] # Г
+0415; [5C 34, 05, 8F] # Е
+
+# tailored (in UK locale) with "Г < ґ <<< Ґ"
+0491; [5C 1A, 05, 09][, DB B9, 09] # ґ
+0490; [5C 1A, 05, 93][, DB B9, 09] # Ґ
+
+# contraction for a tailored collation element
+0491 0306; [5C, DB, 09] # ґ̆
+
+# contractions suppressed in tailoring (for RU locale)
+041A 0301; [5C CC, 05, 8F] # Ќ
+0413 0301; [5C 30, 05, 8F] # Ѓ
+
+# contractions non-suppressed in tailoring
+0415 0306; [5C 36, 05, 8F] # Ӗ
+END
+    end
+
+    let(:tailoring_resource_stub) do
+      <<END
+---
+:tailored_table: ! '0491; [5C1B, 5, 5]
+
+  0490; [5C1B, 5, 86]'
+:suppressed_contractions: ГК
+...
+END
+    end
+
+  end
+
+  # Test data is taken from http://unicode.org/cldr/trac/browser/tags/release-2-0-1/test/
+  # Test files format: # - comments, // - pending tests.
+  #
+  it 'passes tailoring test for each supported locale', :slow => true do
+    TwitterCldr.supported_locales.each do |locale|
+      collator = Collator.new(locale)
+
+      print "#{locale}\t-\t"
+
+      lines = open(File.join(File.dirname(__FILE__), 'tailoring_tests', "#{locale}.txt")) { |f| f.lines.map(&:strip) }
+
+      active_tests  = lines.count(&method(:tailoring_test?))
+      pending_tests = lines.count(&method(:pending_tailoring_test?))
+      print "tests:  %-4d active, %5.1f%% %5s pending\t-\t" % [active_tests, (100.0 * pending_tests / (pending_tests + active_tests)), "(#{pending_tests})"]
+
+      last_number = last = nil
+
+      failures = lines.each_with_index.inject([]) do |memo, (current, number)|
+        if tailoring_test?(current)
+          memo << [last_number + 1, last, current] if tailoring_test?(last) && collator.compare(last, current) == 1
+
+          last = current
+          last_number = number
+        elsif pending_tailoring_test?(current)
+          last_number = last = nil
+        end
+
+        memo
+      end
+
+      if failures.empty?
+        puts "OK"
+      else
+        failures_info = "#{failures.size} failures: #{failures.inspect}"
+
+        puts failures_info
+        failures.should(be_empty, "#{locale} - #{failures_info}")
+      end
+    end
+  end
+
+  def pending_tailoring_test?(line)
+    !!(line =~ %r{^//})
+  end
+
+  def tailoring_test?(line)
+    !!(line && line !~ %r{^(//|#|\s*$)})
+  end
+
+end