twitter_cldr 1.4.1 → 1.5.0

data/spec/collation/collation_spec.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+require 'open-uri'
+require 'zip'
+
+include TwitterCldr::Collation
+
+describe 'Unicode Collation Algorithm' do
+
+  SHORT_COLLATION_TEST_PATH = File.join(File.dirname(__FILE__), 'CollationTest_CLDR_NON_IGNORABLE_Short.txt')
+  FULL_COLLATION_TEST_PATH  = File.join(File.dirname(__FILE__), 'CollationTest_CLDR_NON_IGNORABLE.txt')
+
+  FULL_COLLATION_TEST_URL = 'http://unicode.org/Public/UCA/latest/CollationAuxiliary.zip'
+
+  it 'passes all the tests in CollationTest_CLDR_NON_IGNORABLE_Short.txt' do
+    run_test(SHORT_COLLATION_TEST_PATH)
+  end
+
+  it 'passes all the tests in CollationTest_CLDR_NON_IGNORABLE.txt', :slow => true do
+    prepare_full_test
+    run_test(FULL_COLLATION_TEST_PATH)
+  end
+
+  def run_test(file_path)
+    collator = Collator.new
+
+    previous_sort_key = previous_code_points = previous_hex_code_points = nil
+
+    open(file_path, 'r:utf-8') do |file|
+      file.each do |line|
+        next unless /^([0-9A-F ]+);/ =~ line
+
+        current_code_points = $1.split
+        current_hex_code_points = current_code_points.map { |cp| cp.to_i(16) }
+
+        current_sort_key = collator.sort_key(current_code_points)
+
+        if previous_sort_key
+          result = (previous_sort_key <=> current_sort_key).nonzero? || (previous_hex_code_points <=> current_hex_code_points)
+          result.should(eq(-1), error_message(previous_code_points, previous_sort_key, current_code_points, current_sort_key))
+        end
+
+        previous_sort_key        = current_sort_key
+        previous_code_points     = current_code_points
+        previous_hex_code_points = current_hex_code_points
+      end
+    end
+  end
+
+  # Generates a descriptive error message test failure.
+  #
+  def error_message(previous_code_points, previous_sort_key, current_code_points, current_sort_key)
+<<END
+Expected previous code points sequence to sort before the current one.
+
+  previous:
+    code points - #{previous_code_points.join(' ')}
+    sort key    - #{pretty_sort_key(previous_sort_key)}
+  current:
+    code points - #{current_code_points.join(' ')}
+    sort key    - #{pretty_sort_key(current_sort_key)}
+END
+  end
+
+  # Downloads full version of the test if necessary.
+  #
+  def prepare_full_test
+    return if File.file?(FULL_COLLATION_TEST_PATH)
+
+    print '  Downloading CollationAuxillary.zip ... '
+    zip_file = Tempfile.new('CollationAuxillary.zip')
+    zip_file.write(open(FULL_COLLATION_TEST_URL).read)
+    zip_file.close
+
+    print 'extracting CollationTest_CLDR_NON_IGNORABLE.txt ... '
+    Zip::ZipFile.open(zip_file.path) do |zip|
+      open(FULL_COLLATION_TEST_PATH, 'w') { |file| file.write(zip.read('CollationAuxiliary/CollationTest_CLDR_NON_IGNORABLE.txt')) }
+    end
+    zip_file.unlink
+
+    puts 'done.'
+  end
+
+  def pretty_sort_key(current_sort_key)
+    "[#{current_sort_key.map{ |byte| byte.to_s(16).upcase }.join(', ')}]"
+  end
+
+end

data/spec/collation/collator_spec.rb

@@ -0,0 +1,117 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe Collator do
+
+  before :each do
+    Collator.instance_variable_set(:@trie, nil)
+  end
+
+  after :all do
+    Collator.instance_variable_set(:@trie, nil)
+  end
+
+  describe '.trie' do
+    it 'returns collation elements trie' do
+      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { 'trie' }
+      Collator.trie.should == 'trie'
+    end
+
+    it 'loads the trie only once' do
+      mock(TrieBuilder).load_trie(Collator::FRACTIONAL_UCA_SHORT_RESOURCE) { 'trie' }
+
+      Collator.trie.object_id.should == Collator.trie.object_id
+    end
+  end
+
+  describe '#trie' do
+    it 'delegates to the class method' do
+      mock(Collator).trie { 'trie' }
+      Collator.new.trie.should == 'trie'
+    end
+
+    it 'calls class method only once' do
+      mock(Collator).trie { 'trie' }
+
+      collator = Collator.new
+      collator.trie.object_id.should == collator.trie.object_id
+    end
+  end
+
+  describe '#sort_key' do
+    let(:collator)        { Collator.new }
+    let(:string)          { 'abc' }
+    let(:code_points_hex) { %w[0061 0062 0063] }
+    let(:code_points)     { code_points_hex.map { |cp| cp.to_i(16) } }
+    let(:sort_key)        { [9986, 10498, 11010, 0, 1282, 1282, 1282, 0, 1282, 1282, 1282] }
+
+    before(:each) { mock(collator).sort_key_for_code_points(code_points) { sort_key } }
+
+    it 'calculates sort key for a string' do
+      mock(TwitterCldr::Utils::CodePoints).from_string(string) { code_points_hex }
+      collator.sort_key(string).should == sort_key
+    end
+
+    it 'calculates sort key for an array of code points (represented as hex strings)' do
+      dont_allow(TwitterCldr::Utils::CodePoints).from_string(string)
+      collator.sort_key(code_points_hex).should == sort_key
+    end
+  end
+
+  describe '#compare' do
+    let(:collator)         { Collator.new }
+    let(:sort_key)         { [1, 3, 8, 9] }
+    let(:another_sort_key) { [6, 8, 9, 2] }
+
+    it 'compares strings by sort keys' do
+      stub_sort_key(collator, 'foo', sort_key)
+      stub_sort_key(collator, 'bar', another_sort_key)
+
+      collator.compare('foo', 'bar').should == -1
+      collator.compare('bar', 'foo').should == 1
+    end
+
+    it 'returns 0 without computing sort keys if strings are equal' do
+      dont_allow(collator).sort_key
+
+      collator.compare('foo', 'foo').should == 0
+    end
+
+    it 'compares strings by code points if the sort keys are equal' do
+      stub(collator).sort_key { sort_key }
+
+      collator.compare('bar', 'foo').should == -1
+    end
+  end
+
+  describe '#sort' do
+    let(:collator) { Collator.new }
+
+    it 'sorts strings by sort keys' do
+      [['aaa', [1, 2, 3]], ['abc', [1, 3, 4]], ['bca', [2, 5, 9]]].each { |s, key| mock_sort_key(collator, s, key) }
+
+      collator.sort(%w[bca aaa abc]).should == %w[aaa abc bca]
+    end
+
+    it 'sorts strings with equal sort keys by code points' do
+      [['aaa', [1, 2, 3]], ['abc', [1, 2, 3]], ['bca', [1, 2, 3]]].each { |s, key| mock_sort_key(collator, s, key) }
+
+      collator.sort(%w[bca abc aaa]).should == %w[aaa abc bca]
+    end
+  end
+
+  def mock_sort_key(collator, string, sort_key)
+    mock(collator).sort_key(TwitterCldr::Utils::CodePoints.from_string(string)) { sort_key }
+  end
+
+  def stub_sort_key(collator, string, sort_key)
+    stub(collator).sort_key(TwitterCldr::Utils::CodePoints.from_string(string)) { sort_key }
+  end
+
+end

data/spec/collation/implicit_collation_elements_spec.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe ImplicitCollationElements do
+
+  it 'computes correct implicit value for non-CJK code points' do
+    ImplicitCollationElements.for_code_point(0xD801).should  == [[0xE305C758, 0x5, 0x5]]
+    ImplicitCollationElements.for_code_point(0xC0001).should == [[0xE44E70AC, 0x5, 0x5]]
+    ImplicitCollationElements.for_code_point(0xFFF02).should == [[0xE4C25F74, 0x5, 0x5]]
+  end
+
+  it 'computes correct implicit values for CJK code points' do
+    ImplicitCollationElements.for_code_point(0x4E00).should  == [[0xE00406, 0x5, 0x5]]
+    ImplicitCollationElements.for_code_point(0x3400).should  == [[0xE0ABCE, 0x5, 0x5]]
+    ImplicitCollationElements.for_code_point(0x20000).should == [[0xE1302590, 0x5, 0x5]]
+  end
+
+end

data/spec/collation/sort_key_spec.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe SortKey do
+
+  let(:sort_key) { SortKey.new(collation_elements) }
+  let(:collation_elements) { [[63, 13, 149], [66, 81, 143]] }
+  let(:sort_key_bytes) { [63, 66, 1, 13, 81, 1, 149, 143] }
+
+  describe '.build' do
+    it 'returns a sort key for a given array of collation elements' do
+      sort_key = SortKey.new(collation_elements)
+
+      mock(SortKey).new(collation_elements) { sort_key }
+      mock(sort_key).bytes_array { sort_key_bytes }
+
+      SortKey.build(collation_elements).should == sort_key_bytes
+    end
+  end
+
+  describe '#initialize' do
+    it 'assigns collation elements array' do
+      SortKey.new(collation_elements).collation_elements.should == collation_elements
+    end
+  end
+
+  describe '#bytes_array' do
+    it 'builds sort key bytes' do
+      sort_key.bytes_array.should == sort_key_bytes
+    end
+
+    it 'builds bytes array only once' do
+      mock(sort_key).build_bytes_array { sort_key_bytes }
+      sort_key.bytes_array.object_id == sort_key.bytes_array.object_id
+    end
+
+    it 'compresses secondary weights' do
+      SortKey.new([[0, 5, 0], [0, 5, 0], [0, 141, 0], [0, 5, 0], [0, 5, 0]]).bytes_array.should == [1, 133, 141, 6, 1]
+    end
+
+    it 'compresses tertiary weights' do
+      SortKey.new([[0, 0, 5], [0, 0, 5], [0, 0, 39], [0, 0, 5], [0, 0, 5]]).bytes_array.should == [1, 1, 132, 167, 6]
+    end
+
+    it 'compresses secondary and tertiary weights into multiple bytes if necessary' do
+      SortKey.new([[39, 5, 5]] * 100).bytes_array.should == [39] * 100 + [1, 69, 40, 1, 48, 48, 18]
+    end
+  end
+
+end

data/spec/collation/trie_builder_spec.rb

@@ -0,0 +1,114 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe TrieBuilder do
+
+  describe '#build' do
+    describe 'fractional CE trie hash' do
+      let(:trie_builder) do
+        builder = TrieBuilder.new('resource')
+        stub(builder).load_collation_elements_table { FRACTIONAL_UCA_SHORT_STUB }
+        builder
+      end
+
+      it 'returns a trie' do
+        trie_builder.is_a?(Trie)
+      end
+
+      it 'adds every collation element from the FractionalUCA_SHORT.txt file to the trie' do
+        mock(Trie).new { TrieStub.new }
+
+        trie_builder.build.storage.should == COLLATION_ELEMENTS_TABLE
+      end
+    end
+  end
+
+end
+
+class TrieStub
+  attr_accessor :storage
+
+  def initialize
+    self.storage = []
+  end
+
+  def add(code_points, collation_element)
+    storage << [code_points, collation_element]
+  end
+end
+
+FRACTIONAL_UCA_SHORT_STUB = <<END
+# Fractional UCA Table, generated from standard UCA
+# 2012-01-03, 21:52:55 GMT [MD]
+# VERSION: UCA=6.1.0, UCD=6.1.0
+# For a description of the format and usage, see CollationAuxiliary.html
+
+[UCA version = 6.1.0]
+
+0000; [,,]
+030C; [, 97, 05]
+215E; [20, 05, 3B][0D 75 2C, 05, 3B][22, 05, 3D]
+FC63; [, D3 A9, 33][, D5 11, 33]
+0E40 0E01; [72 0A, 05, 05][72 7E, 05, 3D]
+0E40 0E02; [72 0C, 05, 05][72 7E, 05, 3D]
+
+# HOMELESS COLLATION ELEMENTS
+FDD0 0063;	[, 97, 3D]
+FDD0 0064;	[, A7, 09]
+
+# SPECIAL MAX/MIN COLLATION ELEMENTS
+
+FFFE;	[02, 02, 02]	# Special LOWEST primary, for merge/interleaving
+FFFF;	[EF FE, 05, 05]	# Special HIGHEST primary, for ranges
+
+# Top Byte => Reordering Tokens
+[top_byte	00	TERMINATOR ]	#	[0]	TERMINATOR=1
+[top_byte	01	LEVEL-SEPARATOR ]	#	[0]	LEVEL-SEPARATOR=1
+[top_byte	02	FIELD-SEPARATOR ]	#	[0]	FIELD-SEPARATOR=1
+[top_byte	03	SPACE ]	#	[9]	SPACE=1 Cc=6 Zl=1 Zp=1 Zs=1
+
+# VALUES BASED ON UCA
+[first tertiary ignorable [,,]] # CONSTRUCTED
+[last tertiary ignorable [,,]] # CONSTRUCTED
+# Warning: Case bits are masked in the following
+[first tertiary in secondary non-ignorable [X, X, 05]] # U+0332 COMBINING LOW LINE
+[last tertiary in secondary non-ignorable [X, X, 3D]] # U+2A74 DOUBLE COLON EQUAL
+END
+
+COLLATION_ELEMENTS_TABLE = [
+    # 0000; [,,]
+    [[0], [[0, 0, 0]]],
+
+    # 030C; [, 97, 05]
+    [[780], [[0, 151, 5]]],
+
+    # 215E; [20, 05, 3B][0D 75 2C, 05, 3B][22, 05, 3D]
+    [[8542], [[32, 5, 59], [881964, 5, 59], [34, 5, 61]]],
+
+    # FC63; [, D3 A9, 33][, D5 11, 33]
+    [[64611], [[0, 54185, 51], [0, 54545, 51]]],
+
+    # 0E40 0E01; [72 0A, 05, 05][72 7E, 05, 3D]
+    [[3648, 3585], [[29194, 5, 5], [29310, 5, 61]]],
+
+    # 0E40 0E02; [72 0C, 05, 05][72 7E, 05, 3D]
+    [[3648, 3586], [[29196, 5, 5], [29310, 5, 61]]],
+
+    # FDD0 0063; [, 97, 3D]
+    [[64976, 99], [[0, 151, 61]]],
+
+    # FDD0 0064; [, A7, 09]
+    [[64976, 100], [[0, 167, 9]]],
+
+    # FFFE; [02, 02, 02]
+    [[65534], [[2, 2, 2]]],
+
+    # FFFF; [EF FE, 05, 05]
+    [[65535], [[61438, 5, 5]]]
+]

data/spec/collation/trie_spec.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Collation
+
+describe Trie do
+
+  let(:trie) { Trie.new }
+
+  let(:values) do
+    [
+        [[1],       '1'  ],
+        [[1, 4],    '14' ],
+        [[1, 5],    '15' ],
+        [[1, 4, 8], '148'],
+        [[2],       '2'  ],
+        [[2, 7, 5], '275'],
+        [[3, 9],    '39' ]
+    ]
+  end
+
+  before(:each) do
+    values.each { |key, value| trie.add(key, value) }
+  end
+
+  describe '#get' do
+    it 'returns nil for non existing keys' do
+      [[6], [3], [1, 4, 3], [2, 7, 5, 6, 9]].each { |key| trie.get(key).should be_nil }
+    end
+
+    it 'returns value and key size for each existing key' do
+      values.each { |key, value| trie.get(key).should == value }
+    end
+  end
+
+  describe '#add' do
+    it 'overrides values' do
+      trie.get([1, 4]).should == '14'
+
+      trie.add([1, 4], '14-new')
+      trie.get([1, 4]).should == '14-new'
+    end
+  end
+
+  describe '#find_prefix' do
+    describe 'first (value) and third (prefix size) elements of the returned array' do
+      it 'value is 0 nil and prefix size is 0 if the prefix was not found' do
+        test_find_prefix(trie, [4], nil, 0)
+      end
+
+      it 'stored value and key size as a prefix size if the whole key was found' do
+        values.each do |key, value|
+          test_find_prefix(trie, key, value)
+        end
+      end
+
+      it 'stored value and size of the corresponding prefix if only part of the key was found' do
+        tests = {
+            [1, 9]          => ['1', 1],
+            [1, 4, 2]       => ['14', 2],
+            [1, 4, 8, 9, 2] => ['148', 3],
+            [2, 7, 5, 5]    => ['275', 3]
+        }
+
+        tests.each { |key, result| test_find_prefix(trie, key, *result) }
+      end
+
+      def test_find_prefix(trie, key, value, size = key.size)
+        result = trie.find_prefix(key)
+
+        result[0].should == value
+        result[2].should == size
+      end
+    end
+
+    describe 'second (subtrie) element of the returned array' do
+      it 'is a hash of possible suffixes for the prefix that was found' do
+        trie.find_prefix([1, 4, 8])[1].should == {}
+        trie.find_prefix([2, 7])[1].should == { 5 => ["275", { }] }
+      end
+
+      it 'is a hash representing the whole trie if the prefix was not found' do
+        trie.find_prefix([404])[1].should == {
+            1 => ['1', { 4 => ['14', { 8 => ['148', {}] }], 5 => ['15', {}] }],
+            2 => ['2', { 7 => [nil,  { 5 => ['275', {}] }] }],
+            3 => [nil, { 9 => ['39', {}] }]
+        }
+      end
+    end
+
+  end
+
+end

data/spec/core_ext/calendars/datetime_spec.rb

@@ -35,6 +35,11 @@ describe DateTime do
       loc_date.to_date.calendar_type.should == :buddhist
       loc_date.to_time.calendar_type.should == :buddhist
     end
+
+    it "should default to English if the given locale isn't supported" do
+      loc_date = date.localize(:xx)
+      loc_date.locale.should == :en
+    end
   end
 
   describe "stringify" do

data/spec/core_ext/calendars_spec.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Tokenizers
+
+describe "Calendars" do
+  it "makes sure datetime formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      DateTimeTokenizer::VALID_TYPES.each do |type|
+        lambda { DateTime.now.localize(locale).send(:"to_#{type}_s") }.should_not raise_error
+      end
+    end
+  end
+
+  it "makes sure date formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      DateTimeTokenizer::VALID_TYPES.each do |type|
+        lambda { Date.today.localize(locale).send(:"to_#{type}_s") }.should_not raise_error
+      end
+    end
+  end
+
+  it "makes sure time formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      DateTimeTokenizer::VALID_TYPES.each do |type|
+        lambda { Time.now.localize(locale).send(:"to_#{type}_s") }.should_not raise_error
+      end
+    end
+  end
+end

data/spec/core_ext/numbers_spec.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr
+
+describe "Numbers" do
+  it "makes sure currency formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      lambda { 1337.localize(locale).to_currency.to_s }.should_not raise_error
+      lambda { 1337.localize(locale).to_currency.to_s(:precision => 3) }.should_not raise_error
+      lambda { 1337.localize(locale).to_currency.to_s(:precision => 3, :currency => "EUR") }.should_not raise_error
+    end
+  end
+
+  it "makes sure decimal formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      lambda { 1337.localize(locale).to_decimal.to_s }.should_not raise_error
+      lambda { 1337.localize(locale).to_decimal.to_s(:precision => 3) }.should_not raise_error
+    end
+  end
+
+  it "makes sure percentage formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      lambda { 1337.localize(locale).to_percent.to_s }.should_not raise_error
+      lambda { 1337.localize(locale).to_percent.to_s(:precision => 3) }.should_not raise_error
+    end
+  end
+
+  it "makes sure basic number formatters for every locale don't raise errors" do
+    TwitterCldr.supported_locales.each do |locale|
+      lambda { 1337.localize(locale).to_s }.should_not raise_error
+      lambda { 1337.localize(locale).to_s(:precision => 3) }.should_not raise_error
+    end
+  end
+end

data/spec/core_ext/string_spec.rb

@@ -15,8 +15,8 @@ describe String do
     end
 
     it "uses default locale if it's not explicitly specified" do
-      mock(TwitterCldr).get_locale { :jp }
-      'foo'.localize.locale.should == :jp
+      mock(TwitterCldr).get_locale { :ja }
+      'foo'.localize.locale.should == :ja
     end
 
     it 'uses provided locale if there is one' do
@@ -111,7 +111,7 @@ describe LocalizedString do
 
   describe "#normalize" do
     it "returns a normalized instance of LocalizedString, defaults to NFD" do
-      mock.proxy(TwitterCldr::Normalizers::NFD).normalize("español")
+      mock.proxy(TwitterCldr::Normalization::NFD).normalize("español")
       "español".bytes.to_a.should == [101, 115, 112, 97, 195, 177, 111, 108]
       result = "español".localize.normalize
       result.should be_a(LocalizedString)
@@ -119,7 +119,7 @@ describe LocalizedString do
     end
 
     it "returns a normalized instance of LocalizedString using the specified algorithm" do
-      mock.proxy(TwitterCldr::Normalizers::NFKD).normalize("español")
+      mock.proxy(TwitterCldr::Normalization::NFKD).normalize("español")
       "español".bytes.to_a.should == [101, 115, 112, 97, 195, 177, 111, 108]
       result = "español".localize.normalize(:using => :NFKD)
       result.should be_a(LocalizedString)