twitter_cldr 1.6.1 → 1.6.2

data/spec/collation/collator_spec.rb

@@ -11,53 +11,52 @@ describe Collator do
 
   let(:trie) { Trie.new }
 
-  before(:each) { clear_fce_tries_cache }
-  after(:all)   { clear_fce_tries_cache }
+  before(:each) { clear_tries_cache }
+  after(:all)   { clear_tries_cache }
 
-  describe '.default_fce_trie' do
+  describe '.default_trie' do
     before(:each) do
-      clear_default_fce_trie_cache
-      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { trie }
+      clear_default_trie_cache
+      mock(TrieLoader).load_default_trie { trie }
     end
 
     it 'returns default fractional collation elements trie' do
-      Collator.default_fce_trie.should == trie
+      Collator.default_trie.should == trie
     end
 
     it 'loads the trie only once' do
-      Collator.default_fce_trie.object_id.should == Collator.default_fce_trie.object_id
+      Collator.default_trie.object_id.should == Collator.default_trie.object_id
     end
 
     it 'locks the trie' do
-      Collator.default_fce_trie.should be_locked
+      Collator.default_trie.should be_locked
     end
   end
 
-  describe '.tailored_fce_trie' do
+  describe '.tailored_trie' do
     let(:locale) { :ru }
 
     before(:each) do
-      clear_tailored_fce_tries_cache
-      stub(Collator).default_fce_trie { trie }
-      mock(TrieBuilder).load_tailored_trie(locale, Collator.default_fce_trie) { trie }
+      clear_tailored_tries_cache
+      stub(Collator).default_trie { trie }
+      mock(TrieLoader).load_tailored_trie(locale, Collator.default_trie) { trie }
     end
 
     it 'returns default fractional collation elements trie' do
-      Collator.tailored_fce_trie(locale).should == trie
+      Collator.tailored_trie(locale).should == trie
     end
 
     it 'loads the trie only once' do
-      Collator.tailored_fce_trie(locale).object_id.should == Collator.tailored_fce_trie(locale).object_id
+      Collator.tailored_trie(locale).object_id.should == Collator.tailored_trie(locale).object_id
     end
 
     it 'locks the trie' do
-      Collator.tailored_fce_trie(locale).should be_locked
+      Collator.tailored_trie(locale).should be_locked
     end
   end
 
   describe '#initialize' do
     before :each do
-      stub(TrieBuilder).load_trie { trie }
       any_instance_of(Collator) { |c| stub(c).load_trie { trie } }
     end
 
@@ -106,7 +105,7 @@ describe Collator do
     let(:collation_elements) { [[39, 5, 5], [41, 5, 5], [43, 5, 5]] }
     let(:sort_key)           { [39, 41, 43, 1, 7, 1, 7] }
 
-    before(:each) { stub(TrieBuilder).load_trie { trie } }
+    before(:each) { mock(TrieLoader).load_default_trie { trie } }
 
     describe 'calculating sort key' do
       before(:each) { mock(TwitterCldr::Collation::SortKeyBuilder).build(collation_elements, nil) { sort_key } }
@@ -127,7 +126,7 @@ describe Collator do
       let(:locale)     { :uk }
 
       it 'passes case-first sort option to sort key builder' do
-        mock(TwitterCldr::Collation::TrieBuilder).load_tailored_trie(locale, trie) { Trie.new }
+        mock(TwitterCldr::Collation::TrieLoader).load_tailored_trie(locale, trie) { Trie.new }
         mock(TwitterCldr::Collation::TrieBuilder).tailoring_data(locale) { { :collator_options => { :case_first => case_first } } }
 
         collator = Collator.new(locale)
@@ -145,7 +144,7 @@ describe Collator do
     let(:sort_key)         { [1, 3, 8, 9] }
     let(:another_sort_key) { [6, 8, 9, 2] }
 
-    before(:each) { stub(Collator).default_fce_trie { trie } }
+    before(:each) { stub(Collator).default_trie { trie } }
 
     it 'compares strings by sort keys' do
       stub_sort_key(collator, 'foo', sort_key)
@@ -169,7 +168,7 @@ describe Collator do
     let(:sorted)    { %w[aaa abc bca] }
 
     before :each do
-      stub(Collator).default_fce_trie { trie }
+      stub(Collator).default_trie { trie }
       sort_keys.each { |s, key| mock_sort_key(collator, s, key) }
     end
 
@@ -193,9 +192,16 @@ describe Collator do
 
   describe 'tailoring support' do
     before(:each) do
-      stub(Collator).default_fce_trie { TrieBuilder.parse_trie(fractional_uca_short_stub) }
-      stub(TwitterCldr::Normalization::NFD).normalize_code_points { |code_points| code_points }
       stub(TwitterCldr).get_resource(:collation, :tailoring, locale) { YAML.load(tailoring_resource_stub) }
+
+      mock(File).open(TrieBuilder::FRACTIONAL_UCA_SHORT_PATH, 'r') do |*args|
+        args.last.call(fractional_uca_short_stub)
+      end
+
+      mock(TrieLoader).load_default_trie { TrieBuilder.load_default_trie }
+      mock(TrieLoader).load_tailored_trie.with_any_args { |*args| TrieBuilder.load_tailored_trie(*args) }
+
+      stub(TwitterCldr::Normalization::NFD).normalize_code_points { |code_points| code_points }
     end
 
     let(:locale)            { :some_locale }
@@ -231,7 +237,7 @@ describe Collator do
 
     let(:fractional_uca_short_stub) do
 <<END
-# collation elements from default FCE table
+# collation elements from default fractional collation elements table
 0301; [, 8D, 05]
 0306; [, 91, 05]
 041A; [5C 6C, 05, 8F] # К
@@ -275,17 +281,17 @@ END
     stub(collator).get_sort_key(string) { sort_key }
   end
 
-  def clear_fce_tries_cache
-    clear_default_fce_trie_cache
-    clear_tailored_fce_tries_cache
+  def clear_tries_cache
+    clear_default_trie_cache
+    clear_tailored_tries_cache
   end
 
-  def clear_default_fce_trie_cache
-    Collator.instance_variable_set(:@default_fce_trie, nil)
+  def clear_default_trie_cache
+    Collator.instance_variable_set(:@default_trie, nil)
   end
 
-  def clear_tailored_fce_tries_cache
-    Collator.instance_variable_set(:@tailored_fce_tries_cache, nil)
+  def clear_tailored_tries_cache
+    Collator.instance_variable_set(:@tailored_tries_cache, nil)
   end
 
 end

data/spec/collation/tailoring_spec.rb

@@ -18,7 +18,9 @@ describe 'Unicode collation tailoring' do
 
       print "#{locale}\t-\t"
 
-      lines = open(File.join(File.dirname(__FILE__), 'tailoring_tests', "#{locale}.txt")) { |f| f.lines.map(&:strip) }
+      lines = File.open(File.join(File.dirname(__FILE__), 'tailoring_tests', "#{locale}.txt")) do |file|
+        file.lines.map(&:strip)
+      end
 
       active_tests  = lines.count(&method(:tailoring_test?))
       pending_tests = lines.count(&method(:pending_tailoring_test?))

data/spec/collation/tailoring_tests/ja.txt

@@ -3,11 +3,12 @@
 #   [[2, "x　", "x"], [5, "X　", "X"]] - failures identical to ICU
 #
 #   [156, "xゞx", "xヽ"]] - character 'ゞ', code point 0x309E, is not in NFD (its normalized version is 0x309D 0x3099),
-#   but there is an entry for denormalized version of this string in FCE table - 309E; [0E 25, 05, 05][, DA 95, 05].
-#   As all strings are normalized first, we don't use this entry, but instead build collation elements for this
-#   character from CE's for 0x309D and 0x3099 that are [0E 25, 05, 05] and [, DA 95, 05]. That doesn't cause any issue
-#   in the default locale, because the results are identical. But when 'ゝ' (code point 0x309D) is tailored from
-#   [0E 25, 05, 05] to [0E 29, 5, 5] in JA locale we get wrong [0E 29, 05, 05][, DA 95, 05] collation elements for 'ゞ'.
+#   but there is an entry for denormalized version of this string in fractional collation elements table -
+#   309E; [0E 25, 05, 05][, DA 95, 05]. As all strings are normalized first, we don't use this entry, but instead build
+#   collation elements for this character from CE's for 0x309D and 0x3099 that are [0E 25, 05, 05] and [, DA 95, 05].
+#   That doesn't cause any issue in the default locale, because the results are identical. But when 'ゝ' (code point 0x309D)
+#   is tailored from [0E 25, 05, 05] to [0E 29, 5, 5] in JA locale we get wrong [0E 29, 05, 05][, DA 95, 05] collation
+#   elements for 'ゞ'.
 #
 
 x￣

data/spec/collation/trie_builder_spec.rb

@@ -9,24 +9,19 @@ include TwitterCldr::Collation
 
 describe TrieBuilder do
 
-  describe '.parse_trie' do
-    it 'returns a trie' do
-      TrieBuilder.parse_trie(fractional_uca_short_stub).should be_instance_of(Trie)
-    end
+  describe '.load_default_trie' do
+    let(:trie)  { TrieBuilder.load_default_trie }
 
-    it 'adds every collation element from the FCE table to the trie' do
-      trie = Object.new
-      mock(Trie).new { trie }
-      collation_elements_table.each { |code_points, collation_elements| mock(trie).set(code_points, collation_elements) }
+    before(:each) { mock_default_table }
 
-      TrieBuilder.parse_trie(fractional_uca_short_stub).should == trie
+    it 'returns a Trie' do
+      trie.should be_instance_of(Trie)
     end
 
-    it 'populates the trie that is passed as an argument' do
-      trie = Object.new
-      collation_elements_table.each { |code_points, collation_elements| mock(trie).set(code_points, collation_elements) }
-
-      TrieBuilder.parse_trie(fractional_uca_short_stub, trie).should == trie
+    it 'adds every collation element from the fractional collation elements table to the trie' do
+      collation_elements_table.each do |code_points, collation_elements|
+        trie.get(code_points).should == collation_elements
+      end
     end
 
     let(:fractional_uca_short_stub) do
@@ -104,15 +99,6 @@ END
     end
   end
 
-  describe '.load_trie' do
-    it 'load FCE table from the resource into a trie' do
-      mock(TrieBuilder).parse_trie('fce-table') { 'trie' }
-      mock(TrieBuilder).load_resource('resource') { 'fce-table' }
-
-      TrieBuilder.load_trie('resource').should == 'trie'
-    end
-  end
-
   let(:tailoring_resource_stub) do
 <<END
 ---
@@ -130,10 +116,13 @@ END
 
   describe '.load_tailored_trie' do
     let(:locale)        { :xxx }
-    let(:fallback)      { TrieBuilder.parse_trie(fractional_uca_short_stub) }
+    let(:fallback)      { TrieBuilder.load_default_trie }
     let(:tailored_trie) { TrieBuilder.load_tailored_trie(locale, fallback) }
 
-    before(:each) { mock(TrieBuilder).tailoring_data(locale) { tailoring_data } }
+    before :each do
+      mock_default_table
+      mock(TrieBuilder).tailoring_data(locale) { tailoring_data }
+    end
 
     it 'returns a TrieWithFallback' do
       tailored_trie.should be_instance_of(TrieWithFallback)
@@ -171,7 +160,7 @@ END
 
     let(:fractional_uca_short_stub) do
 <<END
-# collation elements from default FCE table
+# collation elements from default fractional collation elements table
 0301; [, 8D, 05]
 0306; [, 91, 05]
 041A; [5C 6C, 05, 8F] # К
@@ -205,4 +194,10 @@ END
     end
   end
 
+  def mock_default_table
+    mock(File).open(TrieBuilder::FRACTIONAL_UCA_SHORT_PATH, 'r') do |*args|
+      args.last.call(fractional_uca_short_stub)
+    end
+  end
+
 end

data/spec/collation/trie_dumps_spec.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe 'trie dumps', :slow => true do
+  let(:default_trie) { TrieBuilder.load_default_trie }
+  let(:error_message)    { 'expected trie dump to be up-to-date.' }
+
+  it 'has a valid default Fractional Collation Elements trie dump' do
+    TrieLoader.load_default_trie.to_hash.should(eq(default_trie.to_hash), error_message)
+  end
+
+  TwitterCldr.supported_locales.each do |locale|
+    it "has a valid tailored trie dump for #{locale} locale" do
+      loaded_trie = TrieLoader.load_tailored_trie(locale, Trie.new)
+      fresh_trie  = TrieBuilder.load_tailored_trie(locale, default_trie)
+
+      loaded_trie.to_hash.should(eq(fresh_trie.to_hash), error_message)
+    end
+  end
+end

data/spec/collation/trie_loader_spec.rb

@@ -0,0 +1,72 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe TrieLoader do
+
+  describe '.load_default_trie' do
+    let(:root)        { 42 }
+    let(:trie)        { Trie.new(root).tap { |t| t.lock } }
+    let(:trie_dump)   { Marshal.dump(trie) }
+    let(:loaded_trie) { TrieLoader.load_default_trie }
+    let(:locale)      { TrieLoader::DEFAULT_TRIE_LOCALE }
+
+    before(:each) { mock_trie_dump }
+
+    it 'loads a Trie' do
+      loaded_trie.should be_instance_of(Trie)
+    end
+
+    it 'loads trie root' do
+      loaded_trie.instance_variable_get(:@root).should == root
+    end
+
+    it 'loads trie as unlocked' do
+      loaded_trie.should_not be_locked
+    end
+  end
+
+  describe '.load_tailored_trie' do
+    let(:root)              { 42 }
+    let(:original_fallback) { 13 }
+    let(:new_fallback)      { 37 }
+    let(:trie)              { TrieWithFallback.new(original_fallback).tap { |t| t.lock.instance_variable_set(:@root, root) } }
+    let(:locale)            { :ru }
+    let(:trie_dump)         { Marshal.dump(trie) }
+    let(:loaded_trie)       { TrieLoader.load_tailored_trie(locale, new_fallback) }
+
+    before(:each) { mock_trie_dump }
+
+    it 'loads a TrieWithFallback' do
+      loaded_trie.should be_instance_of(TrieWithFallback)
+    end
+
+    it 'loads trie root' do
+      loaded_trie.instance_variable_get(:@root).should == root
+    end
+
+    it 'loads trie as unlocked' do
+      loaded_trie.should_not be_locked
+    end
+
+    it 'sets trie fallback' do
+      loaded_trie.fallback.should == new_fallback
+    end
+  end
+
+  describe '.dump_path' do
+    it 'returns path to a trie dump for a specific locale' do
+      TrieLoader.dump_path(:foo).should == File.join(TwitterCldr::RESOURCES_DIR, 'collation', 'tries', 'foo.dump')
+    end
+  end
+
+  def mock_trie_dump
+    mock(File).open(TrieLoader.dump_path(locale), 'r') { |*args| args.last.call(trie_dump) }
+  end
+
+end

data/spec/collation/trie_spec.rb

@@ -225,6 +225,20 @@ describe Trie do
 
   end
 
+  describe 'marshaling' do
+    it 'dumps trie structure' do
+      trie = Trie.new
+      trie.add([13, 37], 1337)
+      trie.add([42], 42)
+
+      Marshal.load(Marshal.dump(trie)).to_hash.should == { 42 => [42, {}], 13 => [nil, { 37 => [1337, {}] }] }
+    end
+
+    it 'does not dump locked state' do
+      Marshal.load(Marshal.dump(Trie.new.lock)).should_not be_locked
+    end
+  end
+
   describe Trie::Node do
 
     let(:node)          { Trie::Node.new }

data/spec/collation/trie_with_fallback_spec.rb

@@ -38,4 +38,10 @@ describe TrieWithFallback do
     end
   end
 
+  describe 'marshaling' do
+    it 'does not dump fallback' do
+      Marshal.load(Marshal.dump(TrieWithFallback.new(Trie.new))).fallback.should be_nil
+    end
+  end
+
 end

data/spec/normalization/normalization_spec.rb

@@ -59,7 +59,7 @@ describe 'Unicode Normalization Algorithms' do
   # normalization function. Note, how expectation and tests columns indexes match the numbers in the `invariants` hash.
   #
   def run_test(normalizer, invariants, file_path)
-    open(file_path, 'r:UTF-8') do |file|
+    File.open(file_path, 'r:UTF-8') do |file|
       file.each do |line|
         next if line.empty? || line =~ /^(@|#)/
 
@@ -98,7 +98,7 @@ Got:        #{normalized.inspect}
     return if File.file?(FULL_NORMALIZATION_TEST_PATH)
 
     print '    Downloading NormalizationTest.txt ... '
-    open(FULL_NORMALIZATION_TEST_PATH, 'w') { |file| file.write(open(FULL_NORMALIZATION_TEST_URL).read) }
+    File.open(FULL_NORMALIZATION_TEST_PATH, 'w') { |file| file.write(open(FULL_NORMALIZATION_TEST_URL).read) }
     puts 'done.'
   end
 

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: twitter_cldr
 version: !ruby/object:Gem::Version 
-  hash: 13
   prerelease: 
-  segments: 
-  - 1
-  - 6
-  - 1
-  version: 1.6.1
+  version: 1.6.2
 platform: ruby
 authors: 
 - Cameron Dutro
@@ -15,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-19 00:00:00 Z
+date: 2012-07-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: json
@@ -25,11 +20,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        - 1
-        - 9
         version: 1.1.9
   type: :runtime
   version_requirements: *id001
@@ -66,6 +56,7 @@ files:
 - lib/twitter_cldr/collation/sort_key_builder.rb
 - lib/twitter_cldr/collation/trie.rb
 - lib/twitter_cldr/collation/trie_builder.rb
+- lib/twitter_cldr/collation/trie_loader.rb
 - lib/twitter_cldr/collation/trie_with_fallback.rb
 - lib/twitter_cldr/collation.rb
 - lib/twitter_cldr/core_ext/array.rb
@@ -105,9 +96,9 @@ files:
 - lib/twitter_cldr/normalization/nfkc.rb
 - lib/twitter_cldr/normalization/nfkd.rb
 - lib/twitter_cldr/normalization.rb
-- lib/twitter_cldr/resources/import/tailoring.rb
-- lib/twitter_cldr/resources/import.rb
 - lib/twitter_cldr/resources/loader.rb
+- lib/twitter_cldr/resources/tailoring_importer.rb
+- lib/twitter_cldr/resources/tries_dumper.rb
 - lib/twitter_cldr/resources.rb
 - lib/twitter_cldr/shared/calendar.rb
 - lib/twitter_cldr/shared/code_point.rb
@@ -171,6 +162,8 @@ files:
 - spec/collation/tailoring_tests/zh-Hant.txt
 - spec/collation/tailoring_tests/zh.txt
 - spec/collation/trie_builder_spec.rb
+- spec/collation/trie_dumps_spec.rb
+- spec/collation/trie_loader_spec.rb
 - spec/collation/trie_spec.rb
 - spec/collation/trie_with_fallback_spec.rb
 - spec/core_ext/array_spec.rb
@@ -255,6 +248,41 @@ files:
 - resources/collation/tailoring/ur.yml
 - resources/collation/tailoring/zh-Hant.yml
 - resources/collation/tailoring/zh.yml
+- resources/collation/tries/af.dump
+- resources/collation/tries/ar.dump
+- resources/collation/tries/ca.dump
+- resources/collation/tries/cs.dump
+- resources/collation/tries/da.dump
+- resources/collation/tries/de.dump
+- resources/collation/tries/default.dump
+- resources/collation/tries/el.dump
+- resources/collation/tries/en.dump
+- resources/collation/tries/es.dump
+- resources/collation/tries/eu.dump
+- resources/collation/tries/fa.dump
+- resources/collation/tries/fi.dump
+- resources/collation/tries/fil.dump
+- resources/collation/tries/fr.dump
+- resources/collation/tries/he.dump
+- resources/collation/tries/hi.dump
+- resources/collation/tries/hu.dump
+- resources/collation/tries/id.dump
+- resources/collation/tries/it.dump
+- resources/collation/tries/ja.dump
+- resources/collation/tries/ko.dump
+- resources/collation/tries/ms.dump
+- resources/collation/tries/nb.dump
+- resources/collation/tries/nl.dump
+- resources/collation/tries/pl.dump
+- resources/collation/tries/pt.dump
+- resources/collation/tries/ru.dump
+- resources/collation/tries/sv.dump
+- resources/collation/tries/th.dump
+- resources/collation/tries/tr.dump
+- resources/collation/tries/uk.dump
+- resources/collation/tries/ur.dump
+- resources/collation/tries/zh-Hant.dump
+- resources/collation/tries/zh.dump
 - resources/custom/locales/af/units.yml
 - resources/custom/locales/ar/units.yml
 - resources/custom/locales/ca/units.yml
@@ -704,23 +732,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: Text formatting using data from Unicode's Common Locale Data Repository (CLDR).