twitter_cldr 1.6.0 → 1.6.1

data/spec/collation/collator_spec.rb

@@ -56,19 +56,25 @@ describe Collator do
   end
 
   describe '#initialize' do
-    before(:each) { stub(TrieBuilder).load_trie { trie } }
-    before(:each) { any_instance_of(Collator) { |c| stub(c).load_trie { trie } } }
-
-    it 'initializes default collator if locale is not specified' do
-      Collator.new.locale.should be_nil
+    before :each do
+      stub(TrieBuilder).load_trie { trie }
+      any_instance_of(Collator) { |c| stub(c).load_trie { trie } }
     end
 
-    it 'initialized tailored collator if locale is provided' do
-      Collator.new(:ru).locale.should == :ru
+    context 'without locale' do
+      it 'initializes default collator' do
+        Collator.new.locale.should be_nil
+      end
     end
 
-    it 'converts locale' do
-      Collator.new(:no).locale.should == :nb
+    context 'with locale' do
+      it 'initialized tailored collator with provided locale' do
+        Collator.new(:ru).locale.should == :ru
+      end
+
+      it 'converts locale' do
+        Collator.new(:no).locale.should == :nb
+      end
     end
   end
 
@@ -101,16 +107,36 @@ describe Collator do
     let(:sort_key)           { [39, 41, 43, 1, 7, 1, 7] }
 
     before(:each) { stub(TrieBuilder).load_trie { trie } }
-    before(:each) { mock(TwitterCldr::Collation::SortKeyBuilder).build(collation_elements) { sort_key } }
 
-    it 'calculates sort key for a string' do
-      mock(collator).get_collation_elements(string) { collation_elements }
-      collator.get_sort_key(string).should == sort_key
+    describe 'calculating sort key' do
+      before(:each) { mock(TwitterCldr::Collation::SortKeyBuilder).build(collation_elements, nil) { sort_key } }
+
+      it 'calculates sort key for a string' do
+        mock(collator).get_collation_elements(string) { collation_elements }
+        collator.get_sort_key(string).should == sort_key
+      end
+
+      it 'calculates sort key for an array of code points (represented as hex strings)' do
+        mock(collator).get_collation_elements(code_points_hex) { collation_elements }
+        collator.get_sort_key(code_points_hex).should == sort_key
+      end
     end
 
-    it 'calculates sort key for an array of code points (represented as hex strings)' do
-      mock(collator).get_collation_elements(code_points_hex) { collation_elements }
-      collator.get_sort_key(code_points_hex).should == sort_key
+    describe 'uses tailoring options' do
+      let(:case_first) { :upper }
+      let(:locale)     { :uk }
+
+      it 'passes case-first sort option to sort key builder' do
+        mock(TwitterCldr::Collation::TrieBuilder).load_tailored_trie(locale, trie) { Trie.new }
+        mock(TwitterCldr::Collation::TrieBuilder).tailoring_data(locale) { { :collator_options => { :case_first => case_first } } }
+
+        collator = Collator.new(locale)
+
+        mock(collator).get_collation_elements(code_points_hex) { collation_elements }
+        mock(TwitterCldr::Collation::SortKeyBuilder).build(collation_elements, case_first) { sort_key }
+
+        collator.get_sort_key(code_points_hex).should == sort_key
+      end
     end
   end
 
@@ -165,6 +191,82 @@ describe Collator do
     end
   end
 
+  describe 'tailoring support' do
+    before(:each) do
+      stub(Collator).default_fce_trie { TrieBuilder.parse_trie(fractional_uca_short_stub) }
+      stub(TwitterCldr::Normalization::NFD).normalize_code_points { |code_points| code_points }
+      stub(TwitterCldr).get_resource(:collation, :tailoring, locale) { YAML.load(tailoring_resource_stub) }
+    end
+
+    let(:locale)            { :some_locale }
+    let(:default_collator)  { Collator.new }
+    let(:tailored_collator) { Collator.new(locale) }
+
+    describe 'tailoring rules support' do
+      it 'tailored collation elements are used' do
+        default_collator.get_collation_elements(%w[0490]).should  == [[0x5C1A, 5, 0x93], [0, 0xDBB9, 9]]
+        tailored_collator.get_collation_elements(%w[0490]).should == [[0x5C1B, 5, 0x86]]
+
+        default_collator.get_collation_elements(%w[0491]).should  == [[0x5C1A, 5, 9], [0, 0xDBB9, 9]]
+        tailored_collator.get_collation_elements(%w[0491]).should == [[0x5C1B, 5, 5]]
+      end
+
+      it 'original contractions for tailored elements are applied' do
+        default_collator.get_collation_elements(%w[0491 0306]).should  == [[0x5C, 0xDB, 9]]
+        tailored_collator.get_collation_elements(%w[0491 0306]).should == [[0x5C, 0xDB, 9]]
+      end
+    end
+
+    describe 'contractions suppressing support' do
+      it 'suppressed contractions are ignored' do
+        default_collator.get_collation_elements(%w[041A 0301]).should  == [[0x5CCC, 5, 0x8F]]
+        tailored_collator.get_collation_elements(%w[041A 0301]).should == [[0x5C6C, 5, 0x8F], [0, 0x8D, 5]]
+      end
+
+      it 'non-suppressed contractions are used' do
+        default_collator.get_collation_elements(%w[0415 0306]).should  == [[0x5C36, 5, 0x8F]]
+        tailored_collator.get_collation_elements(%w[0415 0306]).should == [[0x5C36, 5, 0x8F]]
+      end
+    end
+
+    let(:fractional_uca_short_stub) do
+<<END
+# collation elements from default FCE table
+0301; [, 8D, 05]
+0306; [, 91, 05]
+041A; [5C 6C, 05, 8F] # К
+0413; [5C 1A, 05, 8F] # Г
+0415; [5C 34, 05, 8F] # Е
+
+# tailored (in UK locale) with "Г < ґ <<< Ґ"
+0491; [5C 1A, 05, 09][, DB B9, 09] # ґ
+0490; [5C 1A, 05, 93][, DB B9, 09] # Ґ
+
+# contraction for a tailored collation element
+0491 0306; [5C, DB, 09] # ґ̆
+
+# contractions suppressed in tailoring (for RU locale)
+041A 0301; [5C CC, 05, 8F] # Ќ
+0413 0301; [5C 30, 05, 8F] # Ѓ
+
+# contractions non-suppressed in tailoring
+0415 0306; [5C 36, 05, 8F] # Ӗ
+END
+    end
+
+    let(:tailoring_resource_stub) do
+<<END
+---
+:tailored_table: ! '0491; [5C1B, 5, 5]
+
+  0490; [5C1B, 5, 86]'
+:suppressed_contractions: ГК
+...
+END
+    end
+
+  end
+
   def mock_sort_key(collator, string, sort_key)
     mock(collator).get_sort_key(string) { sort_key }
   end

data/spec/collation/sort_key_builder_spec.rb

@@ -17,7 +17,7 @@ describe SortKeyBuilder do
     it 'returns a sort key for a given array of collation elements' do
       sort_key = SortKeyBuilder.new(collation_elements)
 
-      mock(SortKeyBuilder).new(collation_elements) { sort_key }
+      mock(SortKeyBuilder).new(collation_elements, nil) { sort_key }
       mock(sort_key).bytes_array { sort_key_bytes }
 
       SortKeyBuilder.build(collation_elements).should == sort_key_bytes
@@ -28,6 +28,16 @@ describe SortKeyBuilder do
     it 'assigns collation elements array' do
       SortKeyBuilder.new(collation_elements).collation_elements.should == collation_elements
     end
+
+    it 'accepts case-first option as the second argument' do
+      SortKeyBuilder::VALID_CASE_FIRST_OPTIONS.each do |case_first|
+        lambda { SortKeyBuilder.new([], case_first) }.should_not raise_error
+      end
+    end
+
+    it 'raises an ArgumentError for invalid case-first option' do
+      lambda { SortKeyBuilder.new([], :wat) }.should raise_error(ArgumentError)
+    end
   end
 
   describe '#bytes_array' do
@@ -40,40 +50,84 @@ describe SortKeyBuilder do
       sort_key.bytes_array.object_id == sort_key.bytes_array.object_id
     end
 
-    it 'compresses primary weights' do
-      SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x908, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
-                          [0x7A, 0x72,           0x73,           0x75, 0x3,    0x9, 0x08,     0x7A, 0x73, 1, 1]
+    describe 'primary weights' do
+      it 'compresses primary weights' do
+        SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x908, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
+                            [0x7A, 0x72,           0x73,           0x75, 0x3,    0x9, 0x08,     0x7A, 0x73, 1, 1]
 
-      SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x9508, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
-                          [0x7A, 0x72,           0x73,           0x75, 0xFF,   0x95, 0x08,     0x7A, 0x73, 1, 1]
-    end
+        SortKeyBuilder.new([[0x7A72,     0, 0], [0x7A73, 0, 0], [0x7A75, 0, 0], [0x9508, 0, 0], [0x7A73, 0, 0]]).bytes_array.should ==
+                            [0x7A, 0x72,           0x73,           0x75, 0xFF,   0x95, 0x08,     0x7A, 0x73, 1, 1]
+      end
 
-    it 'works when there is an ignorable primary weight in the middle' do
-      SortKeyBuilder.new([[0x1312, 0, 0], [0, 0, 0], [0x1415, 0, 0]]).bytes_array.should == [0x13, 0x12, 0x14, 0x15, 1, 1]
-    end
+      it 'works when there is an ignorable primary weight in the middle' do
+        SortKeyBuilder.new([[0x1312, 0, 0], [0, 0, 0], [0x1415, 0, 0]]).bytes_array.should == [0x13, 0x12, 0x14, 0x15, 1, 1]
+      end
 
-    it 'do not compress single byte primary weights' do
-      SortKeyBuilder.new([[0x13, 0, 0], [0x13, 0, 0]]).bytes_array.should == [0x13, 0x13, 1, 1]
-    end
+      it 'do not compress single byte primary weights' do
+        SortKeyBuilder.new([[0x13, 0, 0], [0x13, 0, 0]]).bytes_array.should == [0x13, 0x13, 1, 1]
+      end
 
-    it 'resets primary lead bytes counter after a single byte weight' do
-      SortKeyBuilder.new([[0x1415, 0, 0], [0x13, 0, 0], [0x13, 0, 0], [0x1412, 0, 0]]).bytes_array.should == [0x14, 0x15, 0x13, 0x13, 0x14, 0x12, 1, 1]
-    end
+      it 'resets primary lead bytes counter after a single byte weight' do
+        SortKeyBuilder.new([[0x1415, 0, 0], [0x13, 0, 0], [0x13, 0, 0], [0x1412, 0, 0]]).bytes_array.should == [0x14, 0x15, 0x13, 0x13, 0x14, 0x12, 1, 1]
+      end
 
-    it 'compresses only allowed primary weights' do
-      SortKeyBuilder.new([[0x812, 0, 0], [0x811, 0, 0]]).bytes_array.should == [0x8, 0x12, 0x8, 0x11, 1, 1]
+      it 'compresses only compressible primary weights' do
+        SortKeyBuilder.new([[0x812, 0, 0], [0x811, 0, 0]]).bytes_array.should == [0x8, 0x12, 0x8, 0x11, 1, 1]
+      end
     end
 
-    it 'compresses secondary weights' do
-      SortKeyBuilder.new([[0, 5, 0], [0, 5, 0], [0, 141, 0], [0, 5, 0], [0, 5, 0]]).bytes_array.should == [1, 133, 141, 6, 1]
-    end
+    describe 'secondary weights' do
+      it 'compresses secondary weights' do
+        SortKeyBuilder.new([[0, 5, 0], [0, 5, 0], [0, 141, 0], [0, 5, 0], [0, 5, 0]]).bytes_array.should == [1, 133, 141, 6, 1]
+      end
 
-    it 'compresses tertiary weights' do
-      SortKeyBuilder.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]]).bytes_array.should == [1, 1, 132, 167, 6]
+      it 'compresses secondary weights into multiple bytes if necessary' do
+        SortKeyBuilder.new([[0, 5, 0]] * 100).bytes_array.should == [1, 69, 40, 1]
+      end
     end
 
-    it 'compresses secondary and tertiary weights into multiple bytes if necessary' do
-      SortKeyBuilder.new([[0, 5, 5]] * 100).bytes_array.should == [1, 69, 40, 1, 48, 48, 18]
+    describe 'tertiary weights' do
+      context 'when case_first is not set' do
+        it 'removes case bits and adds top addition to bytes that are greater than common' do
+          SortKeyBuilder.new([[0, 0, 9], [0, 0, 73], [0, 0, 137], [0, 0, 201]]).bytes_array.should == [1, 1, 137, 137, 137, 137]
+        end
+
+        it 'compresses tertiary weights' do
+          SortKeyBuilder.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]]).bytes_array.should == [1, 1, 0x84, 0xA7, 6]
+        end
+
+        it 'compresses tertiary weights into multiple bytes if necessary' do
+          SortKeyBuilder.new([[0, 0, 5]] * 100).bytes_array.should == [1, 1, 0x30, 0x30, 0x12]
+        end
+      end
+
+      context 'when case_first is :upper' do
+        it 'inverts case bits and subtract bottom addition from bytes that are smaller than common' do
+          SortKeyBuilder.new([[0, 0, 9], [0, 0, 80], [0, 0, 143]], :upper).bytes_array.should == [1, 1, 201, 80, 15]
+        end
+
+        it 'compresses tertiary weights' do
+          SortKeyBuilder.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]], :upper).bytes_array.should == [1, 1, 0xC4, 0xE7, 0xC3]
+        end
+
+        it 'compresses tertiary weights into multiple bytes if necessary' do
+          SortKeyBuilder.new([[0, 0, 5]] * 100, :upper).bytes_array.should == [1, 1, 0x9C, 0x9C, 0xB3]
+        end
+      end
+
+      context 'when case_first is :lower' do
+        it 'leaves case bits and adds top addition to bytes that are greater than common' do
+          SortKeyBuilder.new([[0, 0, 9], [0, 0, 80], [0, 0, 143]], :lower).bytes_array.should == [1, 1, 73, 144, 207]
+        end
+
+        it 'compresses tertiary weights' do
+          SortKeyBuilder.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]], :lower).bytes_array.should == [1, 1, 0x44, 0x67, 6]
+        end
+
+        it 'compresses tertiary weights into multiple bytes if necessary' do
+          SortKeyBuilder.new([[0, 0, 5]] * 100, :lower).bytes_array.should == [1, 1, 0x1A, 0x1A, 0x1A, 0x1A, 0x14]
+        end
+      end
     end
   end
 

data/spec/collation/tailoring_spec.rb

@@ -9,82 +9,6 @@ include TwitterCldr::Collation
 
 describe 'Unicode collation tailoring' do
 
-  describe 'tailoring support' do
-    before(:each) do
-      stub(Collator).default_fce_trie { TrieBuilder.parse_trie(fractional_uca_short_stub) }
-      stub(TwitterCldr::Normalization::NFD).normalize_code_points { |code_points| code_points }
-      stub(TwitterCldr).get_resource(:collation, :tailoring, locale) { YAML.load(tailoring_resource_stub) }
-    end
-
-    let(:locale)            { :some_locale }
-    let(:default_collator)  { Collator.new }
-    let(:tailored_collator) { Collator.new(locale) }
-
-    describe 'tailoring rules support' do
-      it 'tailored collation elements are used' do
-        default_collator.get_collation_elements(%w[0490]).should  == [[0x5C1A, 5, 0x93], [0, 0xDBB9, 9]]
-        tailored_collator.get_collation_elements(%w[0490]).should == [[0x5C1B, 5, 0x86]]
-
-        default_collator.get_collation_elements(%w[0491]).should  == [[0x5C1A, 5, 9], [0, 0xDBB9, 9]]
-        tailored_collator.get_collation_elements(%w[0491]).should == [[0x5C1B, 5, 5]]
-      end
-
-      it 'original contractions for tailored elements are applied' do
-        default_collator.get_collation_elements(%w[0491 0306]).should  == [[0x5C, 0xDB, 9]]
-        tailored_collator.get_collation_elements(%w[0491 0306]).should == [[0x5C, 0xDB, 9]]
-      end
-    end
-
-    describe 'contractions suppressing support' do
-      it 'suppressed contractions are ignored' do
-        default_collator.get_collation_elements(%w[041A 0301]).should  == [[0x5CCC, 5, 0x8F]]
-        tailored_collator.get_collation_elements(%w[041A 0301]).should == [[0x5C6C, 5, 0x8F], [0, 0x8D, 5]]
-      end
-
-      it 'non-suppressed contractions are used' do
-        default_collator.get_collation_elements(%w[0415 0306]).should  == [[0x5C36, 5, 0x8F]]
-        tailored_collator.get_collation_elements(%w[0415 0306]).should == [[0x5C36, 5, 0x8F]]
-      end
-    end
-
-    let(:fractional_uca_short_stub) do
-      <<END
-# collation elements from default FCE table
-0301; [, 8D, 05]
-0306; [, 91, 05]
-041A; [5C 6C, 05, 8F] # К
-0413; [5C 1A, 05, 8F] # Г
-0415; [5C 34, 05, 8F] # Е
-
-# tailored (in UK locale) with "Г < ґ <<< Ґ"
-0491; [5C 1A, 05, 09][, DB B9, 09] # ґ
-0490; [5C 1A, 05, 93][, DB B9, 09] # Ґ
-
-# contraction for a tailored collation element
-0491 0306; [5C, DB, 09] # ґ̆
-
-# contractions suppressed in tailoring (for RU locale)
-041A 0301; [5C CC, 05, 8F] # Ќ
-0413 0301; [5C 30, 05, 8F] # Ѓ
-
-# contractions non-suppressed in tailoring
-0415 0306; [5C 36, 05, 8F] # Ӗ
-END
-    end
-
-    let(:tailoring_resource_stub) do
-      <<END
----
-:tailored_table: ! '0491; [5C1B, 5, 5]
-
-  0490; [5C1B, 5, 86]'
-:suppressed_contractions: ГК
-...
-END
-    end
-
-  end
-
   # Test data is taken from http://unicode.org/cldr/trac/browser/tags/release-2-0-1/test/
   # Test files format: # - comments, // - pending tests.
   #