twitter_cldr 1.5.0 → 1.6.0

data/spec/collation/trie_builder_spec.rb

@@ -9,41 +9,28 @@ include TwitterCldr::Collation
 
 describe TrieBuilder do
 
-  describe '#build' do
-    describe 'fractional CE trie hash' do
-      let(:trie_builder) do
-        builder = TrieBuilder.new('resource')
-        stub(builder).load_collation_elements_table { FRACTIONAL_UCA_SHORT_STUB }
-        builder
-      end
-
-      it 'returns a trie' do
-        trie_builder.is_a?(Trie)
-      end
-
-      it 'adds every collation element from the FractionalUCA_SHORT.txt file to the trie' do
-        mock(Trie).new { TrieStub.new }
-
-        trie_builder.build.storage.should == COLLATION_ELEMENTS_TABLE
-      end
+  describe '#parse_trie' do
+    it 'returns a trie' do
+      TrieBuilder.parse_trie(fractional_uca_short_stub).should be_instance_of(Trie)
     end
-  end
 
-end
+    it 'adds every collation element from the FCE table to the trie' do
+      trie = Object.new
+      mock(Trie).new { trie }
+      collation_elements_table.each { |code_points, collation_elements| mock(trie).set(code_points, collation_elements) }
 
-class TrieStub
-  attr_accessor :storage
+      TrieBuilder.parse_trie(fractional_uca_short_stub).should == trie
+    end
 
-  def initialize
-    self.storage = []
-  end
+    it 'populates the trie that is passed as an argument' do
+      trie = Object.new
+      collation_elements_table.each { |code_points, collation_elements| mock(trie).set(code_points, collation_elements) }
 
-  def add(code_points, collation_element)
-    storage << [code_points, collation_element]
-  end
-end
+      TrieBuilder.parse_trie(fractional_uca_short_stub, trie).should == trie
+    end
 
-FRACTIONAL_UCA_SHORT_STUB = <<END
+    let(:fractional_uca_short_stub) do
+<<END
 # Fractional UCA Table, generated from standard UCA
 # 2012-01-03, 21:52:55 GMT [MD]
 # VERSION: UCA=6.1.0, UCD=6.1.0
@@ -80,35 +67,128 @@ FFFF;	[EF FE, 05, 05]	# Special HIGHEST primary, for ranges
 [first tertiary in secondary non-ignorable [X, X, 05]] # U+0332 COMBINING LOW LINE
 [last tertiary in secondary non-ignorable [X, X, 3D]] # U+2A74 DOUBLE COLON EQUAL
 END
+    end
 
-COLLATION_ELEMENTS_TABLE = [
-    # 0000; [,,]
-    [[0], [[0, 0, 0]]],
+    let(:collation_elements_table) do
+      [
+          # 0000; [,,]
+          [[0], [[0, 0, 0]]],
 
-    # 030C; [, 97, 05]
-    [[780], [[0, 151, 5]]],
+          # 030C; [, 97, 05]
+          [[780], [[0, 151, 5]]],
 
-    # 215E; [20, 05, 3B][0D 75 2C, 05, 3B][22, 05, 3D]
-    [[8542], [[32, 5, 59], [881964, 5, 59], [34, 5, 61]]],
+          # 215E; [20, 05, 3B][0D 75 2C, 05, 3B][22, 05, 3D]
+          [[8542], [[32, 5, 59], [881964, 5, 59], [34, 5, 61]]],
 
-    # FC63; [, D3 A9, 33][, D5 11, 33]
-    [[64611], [[0, 54185, 51], [0, 54545, 51]]],
+          # FC63; [, D3 A9, 33][, D5 11, 33]
+          [[64611], [[0, 54185, 51], [0, 54545, 51]]],
 
-    # 0E40 0E01; [72 0A, 05, 05][72 7E, 05, 3D]
-    [[3648, 3585], [[29194, 5, 5], [29310, 5, 61]]],
+          # 0E40 0E01; [72 0A, 05, 05][72 7E, 05, 3D]
+          [[3648, 3585], [[29194, 5, 5], [29310, 5, 61]]],
 
-    # 0E40 0E02; [72 0C, 05, 05][72 7E, 05, 3D]
-    [[3648, 3586], [[29196, 5, 5], [29310, 5, 61]]],
+          # 0E40 0E02; [72 0C, 05, 05][72 7E, 05, 3D]
+          [[3648, 3586], [[29196, 5, 5], [29310, 5, 61]]],
 
-    # FDD0 0063; [, 97, 3D]
-    [[64976, 99], [[0, 151, 61]]],
+          # FDD0 0063; [, 97, 3D]
+          [[64976, 99], [[0, 151, 61]]],
 
-    # FDD0 0064; [, A7, 09]
-    [[64976, 100], [[0, 167, 9]]],
+          # FDD0 0064; [, A7, 09]
+          [[64976, 100], [[0, 167, 9]]],
+
+          # FFFE; [02, 02, 02]
+          [[65534], [[2, 2, 2]]],
+
+          # FFFF; [EF FE, 05, 05]
+          [[65535], [[61438, 5, 5]]]
+      ]
+    end
+  end
+
+  describe '#load_trie' do
+    it 'load FCE table from the resource into a trie' do
+      mock(TrieBuilder).parse_trie('fce-table') { 'trie' }
+      mock(TrieBuilder).load_resource('resource') { 'fce-table' }
+
+      TrieBuilder.load_trie('resource').should == 'trie'
+    end
+  end
+
+  describe '#load_tailored_trie' do
+    let(:locale)        { :xxx }
+    let(:fallback)      { TrieBuilder.parse_trie(fractional_uca_short_stub) }
+    let(:tailored_trie) { TrieBuilder.load_tailored_trie(locale, fallback) }
+
+    before(:each) { mock(TwitterCldr).get_resource(:collation, :tailoring, locale) { YAML.load(tailoring_resource_stub) } }
+
+    it 'returns a TrieWithFallback' do
+      tailored_trie.should be_instance_of(TrieWithFallback)
+    end
 
-    # FFFE; [02, 02, 02]
-    [[65534], [[2, 2, 2]]],
+    it 'tailors elements in the trie' do
+      fallback.get([0x0491]).should == [[0x5C1A, 5, 9], [0, 0xDBB9, 9]]
+      fallback.get([0x0490]).should == [[0x5C1A, 5, 0x93], [0, 0xDBB9, 9]]
+
+      tailored_trie.get([0x0491]).should == [[0x5C1B, 5, 5]]
+      tailored_trie.get([0x0490]).should == [[0x5C1B, 5, 0x86]]
+    end
+
+    it 'makes contractions available in the tailored trie' do
+      tailored_trie.get([0x491, 0x306]).should == [[0x5C, 0xDB, 9]]
+      tailored_trie.get([0x415, 0x306]).should == [[0x5C36, 5, 0x8F]]
+    end
+
+    it 'suppresses required contractions' do
+      fallback.find_prefix([0x41A, 0x301]).first(2).should == [[[0x5CCC, 5, 0x8F]], 2]
+      fallback.find_prefix([0x413, 0x301]).first(2).should == [[[0x5C30, 5, 0x8F]], 2]
+
+      tailored_trie.find_prefix([0x41A, 0x301]).first(2).should == [[[0x5C6C, 5, 0x8F]], 1]
+      tailored_trie.find_prefix([0x413, 0x301]).first(2).should == [[[0x5C1A, 5, 0x8F]], 1]
+    end
+
+    it 'do not copy other collation elements from the fallback' do
+      %w[0301 0306 041A 0413 0415].each do |code_point|
+        code_points = [code_point.to_i(16)]
+
+        tailored_trie.get(code_points).should_not be_nil
+        tailored_trie.get(code_points).object_id.should == fallback.get(code_points).object_id
+      end
+    end
+
+    let(:fractional_uca_short_stub) do
+<<END
+# collation elements from default FCE table
+0301; [, 8D, 05]
+0306; [, 91, 05]
+041A; [5C 6C, 05, 8F] # К
+0413; [5C 1A, 05, 8F] # Г
+0415; [5C 34, 05, 8F] # Е
+
+# tailored (in UK locale) with "Г < ґ <<< Ґ"
+0491; [5C 1A, 05, 09][, DB B9, 09] # ґ
+0490; [5C 1A, 05, 93][, DB B9, 09] # Ґ
+
+# contraction for a tailored collation element
+0491 0306; [5C, DB, 09] # ґ̆
+
+# contractions suppressed in tailoring (for RU locale)
+041A 0301; [5C CC, 05, 8F] # Ќ
+0413 0301; [5C 30, 05, 8F] # Ѓ
+
+# contractions non-suppressed in tailoring
+0415 0306; [5C 36, 05, 8F] # Ӗ
+END
+    end
+
+    let(:tailoring_resource_stub) do
+<<END
+---
+:tailored_table: ! '0491; [5C1B, 5, 5]
+
+  0490; [5C1B, 5, 86]'
+:suppressed_contractions: ГК
+...
+END
+    end
+  end
 
-    # FFFF; [EF FE, 05, 05]
-    [[65535], [[61438, 5, 5]]]
-]
+end

data/spec/collation/trie_spec.rb

@@ -15,16 +15,67 @@ describe Trie do
     [
         [[1],       '1'  ],
         [[1, 4],    '14' ],
-        [[1, 5],    '15' ],
         [[1, 4, 8], '148'],
+        [[1, 5],    '15' ],
         [[2],       '2'  ],
         [[2, 7, 5], '275'],
-        [[3, 9],    '39' ]
+        [[3, 9, 2], '392'],
+        [[4],       '4'  ]
     ]
   end
 
-  before(:each) do
-    values.each { |key, value| trie.add(key, value) }
+  before(:each) { values.each { |key, value| trie.add(key, value) } }
+
+  describe '#initialize' do
+    it 'initializes an empty trie by default' do
+      Trie.new.should be_empty
+    end
+
+    it 'initializes with a root node' do
+      trie = Trie.new(Trie::Node.new(nil, 1 => Trie::Node.new(nil, { 2 => Trie::Node.new('12')}), 2 => Trie::Node.new('2')))
+
+      trie.to_hash.should == {
+          1 => [nil, { 2 => ['12', {}] }],
+          2 => ['2', {}]
+      }
+    end
+  end
+
+  describe '#lock and #locked?' do
+    it 'trie is unlocked by default' do
+      trie.should_not be_locked
+    end
+
+    it '#lock locks the trie' do
+      trie.lock
+      trie.should be_locked
+    end
+
+    it '#lock returns the trie' do
+      trie.lock.should == trie
+    end
+  end
+
+  describe '#starters' do
+    it 'returns all unique first elements of the keys in the trie' do
+      trie.starters.should =~ [1, 2, 3, 4]
+    end
+  end
+
+  describe '#each_starting_with' do
+    it 'iterates over all key-value pairs for which key starts with a given value' do
+      res = {}
+      trie.each_starting_with(1) { |k, v| res[k] = v }
+
+      res.should == { [1] => '1', [1, 4] => '14', [1, 5] => '15', [1, 4, 8] => '148' }
+    end
+
+    it 'works when argument is not a starter' do
+      res = {}
+      trie.each_starting_with(42) { |k, v| res[k] = v }
+
+      res.should == {}
+    end
   end
 
   describe '#get' do
@@ -32,29 +83,69 @@ describe Trie do
       [[6], [3], [1, 4, 3], [2, 7, 5, 6, 9]].each { |key| trie.get(key).should be_nil }
     end
 
-    it 'returns value and key size for each existing key' do
+    it 'returns value for each existing key' do
       values.each { |key, value| trie.get(key).should == value }
     end
   end
 
   describe '#add' do
-    it 'overrides values' do
+    it 'does not override values' do
       trie.get([1, 4]).should == '14'
 
       trie.add([1, 4], '14-new')
+      trie.get([1, 4]).should == '14'
+    end
+
+    it 'adds new values' do
+      trie.get([1, 9]).should be_nil
+
+      trie.add([1, 9], '19')
+      trie.get([1, 9]).should == '19'
+    end
+
+    it 'raises RuntimeError if called on a locked trie' do
+      lambda { trie.lock.add([1, 3], 'value') }.should raise_error(RuntimeError)
+    end
+  end
+
+  describe '#set' do
+    it 'overrides values' do
+      trie.get([1, 4]).should == '14'
+
+      trie.set([1, 4], '14-new')
       trie.get([1, 4]).should == '14-new'
     end
+
+    it 'adds new values' do
+      trie.get([1, 9]).should be_nil
+
+      trie.set([1, 9], '19')
+      trie.get([1, 9]).should == '19'
+    end
+
+    it 'raises RuntimeError if called on a locked trie' do
+      lambda { trie.lock.set([1, 3], 'value') }.should raise_error(RuntimeError)
+    end
   end
 
   describe '#find_prefix' do
-    describe 'first (value) and third (prefix size) elements of the returned array' do
-      it 'value is 0 nil and prefix size is 0 if the prefix was not found' do
-        test_find_prefix(trie, [4], nil, 0)
+    let(:root_subtrie) {
+      {
+          1 => ['1', { 4 => ['14', { 8 => ['148', {}] }], 5 => ['15', {}] }],
+          2 => ['2', { 7 => [nil, { 5 => ['275', {}] }] }],
+          3 => [nil, { 9 => [nil, { 2 => ['392', {}] }] }],
+          4 => ['4', {}]
+      }
+    }
+
+    describe 'first two elements of the returned array (value and prefix size)' do
+      it 'are nil and 0 if the prefix was not found' do
+        trie.find_prefix([42]).first(2).should == [nil, 0]
       end
 
-      it 'stored value and key size as a prefix size if the whole key was found' do
+      it 'are the stored value and the key size if the whole key was found' do
         values.each do |key, value|
-          test_find_prefix(trie, key, value)
+          trie.find_prefix(key).first(2).should == [value, key.size]
         end
       end
 
@@ -66,32 +157,216 @@ describe Trie do
             [2, 7, 5, 5]    => ['275', 3]
         }
 
-        tests.each { |key, result| test_find_prefix(trie, key, *result) }
+        tests.each do |key, result|
+          trie.find_prefix(key).first(2).should == result
+        end
       end
 
       def test_find_prefix(trie, key, value, size = key.size)
-        result = trie.find_prefix(key)
-
-        result[0].should == value
-        result[2].should == size
+        trie.find_prefix(key).first(2).should == [value, size]
       end
     end
 
-    describe 'second (subtrie) element of the returned array' do
-      it 'is a hash of possible suffixes for the prefix that was found' do
-        trie.find_prefix([1, 4, 8])[1].should == {}
-        trie.find_prefix([2, 7])[1].should == { 5 => ["275", { }] }
+    describe 'last element of the returned array (suffixes subtrie)' do
+      let(:non_existing_key)     { [5, 2, 7] }
+      let(:key_with_suffixes)    { [2] }
+      let(:key_without_suffixes) { [1, 4, 8] }
+
+      it 'is always a locked trie' do
+        [trie, trie.lock].each do |some_trie|
+          [non_existing_key, key_with_suffixes, key_without_suffixes].each do |key|
+            some_trie.find_prefix(key).last.should be_locked
+          end
+        end
+      end
+
+      it 'is a locked empty subtrie if the prefix that was found does not have any suffixes' do
+        trie.find_prefix(key_without_suffixes).last.to_hash.should be_empty
+      end
+
+      it 'is a subtrie of possible suffixes for the prefix that was found' do
+        trie.find_prefix(key_with_suffixes).last.to_hash.should == { 7 => [nil, { 5 => ["275", {}] }] }
       end
 
       it 'is a hash representing the whole trie if the prefix was not found' do
-        trie.find_prefix([404])[1].should == {
-            1 => ['1', { 4 => ['14', { 8 => ['148', {}] }], 5 => ['15', {}] }],
-            2 => ['2', { 7 => [nil,  { 5 => ['275', {}] }] }],
-            3 => [nil, { 9 => ['39', {}] }]
-        }
+        trie.get(non_existing_key).should be_nil
+
+        trie.find_prefix(non_existing_key).last.to_hash.should == root_subtrie
+      end
+
+    end
+
+    context 'argument does not match any value, but is a prefix of a longer key' do
+      context 'argument has a shorter key as a prefix' do
+        it 'returns value for the key, its size and suffixes subtrie' do
+          trie.get([2]).should_not be_nil
+          trie.get([2, 7, 5]).should_not be_nil
+
+          result = trie.find_prefix([2, 7])
+
+          result.first(2).should == ['2', 1]
+          result.last.to_hash.should == { 7 => [nil, { 5 => ["275", {}] }] }
+        end
+      end
+
+      context 'argument does not have a shorter key as a prefix' do
+        it 'returns nil, 0 and suffixes subtrie for the root node' do
+          trie.get([3]).should be_nil
+          trie.get([3, 9]).should be_nil
+          trie.get([3, 9, 2]).should_not be_nil
+
+          result = trie.find_prefix([3, 9])
+
+          result.first(2).should == [nil, 0]
+          result.last.to_hash.should == root_subtrie
+        end
+      end
+    end
+
+  end
+
+  describe Trie::Node do
+
+    let(:node)          { Trie::Node.new }
+    let(:child)         { Trie::Node.new('child') }
+    let(:another_child) { Trie::Node.new('another-child') }
+
+    let(:root_node) do
+      Trie::Node.new(
+          'node-0',
+          1 => Trie::Node.new(
+              'node-1',
+              1 => Trie::Node.new('node-11'),
+              2 => Trie::Node.new('node-12')
+          ),
+          2 => Trie::Node.new(
+              'node-2',
+              1 => Trie::Node.new(
+                  'node-21',
+                  1 => Trie::Node.new('node-211')
+              )
+          )
+      )
+    end
+
+    let(:subtrie_hash) do
+      {
+          1 => [
+              'node-1',
+              {
+                  1 => ['node-11', {}],
+                  2 => ['node-12', {}]
+              }
+          ],
+          2 => [
+              'node-2',
+              {
+                  1 => [
+                      'node-21',
+                      {
+                          1 => ['node-211', {}]
+                      }
+                  ]
+              }
+          ]
+      }
+    end
+
+    describe '#initialize' do
+      it 'initializes node with nil value and empty children hash by default' do
+        node.value.should be_nil
+        node.should_not have_children
+      end
+
+      it 'initializes node with provided value and children hash' do
+        root_node.value.should == 'node-0'
+        root_node.should have_children
+      end
+    end
+
+    describe '#child and #set_child' do
+      it '#child returns nil if a child with a given key does not exist' do
+        node.child(42).should be_nil
+      end
+
+      it '#set_child saves a child by key and #child returns the child by key' do
+        node.set_child(42, child)
+        node.child(42).should == child
+      end
+
+      it '#set_child overrides a child by key' do
+        node.set_child(42, child)
+        node.set_child(42, another_child)
+
+        node.child(42).should_not == child
+        node.child(42).should == another_child
+      end
+
+      it '#set_child returns the child that was saved' do
+        node.set_child(42, child).should == child
+      end
+    end
+
+    describe '#each_key_and_child' do
+      it 'iterates over all (key, child) pairs' do
+        node.set_child(42, child)
+        node.set_child(13, another_child)
+        res = {}
+        node.each_key_and_child { |key, child| res[key] = child }
+
+        res.should == { 42 => child, 13 => another_child }
+      end
+    end
+
+    describe '#keys' do
+      it 'returns all children keys' do
+        node.set_child(42, child)
+        node.set_child(13, another_child)
+
+        node.keys.should =~ [13, 42]
+      end
+    end
+
+    describe '#has_children?' do
+      it 'returns false if the node has no children' do
+        node.should_not have_children
+      end
+
+      it 'returns true if the node has children' do
+        node.set_child(42, child)
+        node.should have_children
+      end
+    end
+
+    describe '#to_trie' do
+      it 'returns a trie' do
+        node.to_trie.should be_instance_of(Trie)
+      end
+
+      it 'returns a locked trie' do
+        node.to_trie.should be_locked
+      end
+
+      it 'current node is a root of a new trie' do
+        root_node.to_trie.to_hash.should == subtrie_hash
+      end
+
+      it 'sets new trie root value to nil' do
+        root_node.value.should_not be_nil
+        root_node.to_trie.instance_variable_get(:@root).value.should be_nil
+      end
+    end
+
+    describe '#subtrie_hash' do
+      it 'returns an empty hash if the node has no children' do
+        node.subtrie_hash.should == {}
+      end
+
+      it 'returns a nested hash of children values' do
+        root_node.subtrie_hash.should == subtrie_hash
       end
     end
 
   end
 
-end
+end