twitter_cldr 3.0.0.beta1 → 3.0.0

data/spec/normalization/base_spec.rb

@@ -1,16 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'spec_helper'
-include TwitterCldr::Normalization
-
-describe Base do
-  describe "#combining_class_for" do
-    it "returns the correct combining class for select code points" do
-      TwitterCldr::Normalization::Base.combining_class_for(0x303).should == 230  # combining tilde
-      TwitterCldr::Normalization::Base.combining_class_for(0x6E).should  == 0    # latin letter n
-    end
-  end
-end

data/spec/normalization/hangul_spec.rb

@@ -1,42 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'spec_helper'
-
-include TwitterCldr::Normalization
-
-describe Hangul do
-
-  describe ".compose" do
-    it 'composes decomposed Hangul syllable without a trailing consonant' do
-      Hangul.compose([0x1101, 0x1167]).should == 0xAEF4
-    end
-
-    it 'composes decomposed Hangul syllable with a trailing consonant' do
-      Hangul.compose([0x1111, 0x1171, 0x11B6]).should == 0xD4DB
-    end
-  end
-
-  describe ".decompose" do
-    it 'decomposes precomposed Hangul syllable without a trailing consonant' do
-      Hangul.decompose(0xAEF4).should == [0x1101, 0x1167]
-    end
-
-    it 'decomposes precomposed Hangul syllable with a trailing consonant' do
-      Hangul.decompose(0xD4DB).should == [0x1111, 0x1171, 0x11B6]
-    end
-  end
-
-  describe '.hangul_syllable?' do
-    it 'returns true for code points from Hangul syllables range' do
-      [0xAC00, 0xAC01, 0xBC9F, 0xD7A1, 0xD7A3].map { |code_point| Hangul.hangul_syllable?(code_point).should be_true }
-    end
-
-    it 'returns false for other code points' do
-      [0xAB, 0xABFF, 0xD7A4, 0xFFCF].map { |code_point| Hangul.hangul_syllable?(code_point).should be_false }
-    end
-  end
-
-end

data/spec/normalization/normalization_spec.rb

@@ -1,113 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'spec_helper'
-
-require 'open-uri'
-
-include TwitterCldr::Normalization
-
-describe 'Unicode Normalization Algorithms' do
-
-  SHORT_NORMALIZATION_TEST_PATH = File.join(File.dirname(__FILE__), 'NormalizationTestShort.txt')
-  FULL_NORMALIZATION_TEST_PATH  = File.join(File.dirname(__FILE__), 'NormalizationTest.txt')
-
-  FULL_NORMALIZATION_TEST_URL = 'http://unicode.org/Public/UNIDATA/NormalizationTest.txt'
-
-  shared_examples_for 'a normalization algorithm' do
-    before(:each) do
-      TwitterCldr::Shared::CodePoint.send(:block_cache).clear
-    end
-
-    it 'passes all the tests in NormalizationTestShort.txt' do
-      run_test(described_class, invariants, SHORT_NORMALIZATION_TEST_PATH)
-    end
-
-    it 'passes all the tests in NormalizationTest.txt', :slow => true do
-      prepare_full_test
-      run_test(described_class, invariants, FULL_NORMALIZATION_TEST_PATH)
-    end
-  end
-
-  describe NFD do
-    let(:invariants) { { 3 => [1, 2, 3], 5 => [4, 5] } }
-    it_behaves_like 'a normalization algorithm'
-  end
-
-  describe NFKD do
-    let(:invariants) { { 5 => [1, 2, 3, 4, 5] } }
-    it_behaves_like 'a normalization algorithm'
-  end
-
-  describe NFC do
-    let(:invariants) { { 2 => [1, 2, 3], 4 => [4, 5] } }
-    it_behaves_like 'a normalization algorithm'
-  end
-
-  describe NFKC do
-    let(:invariants) { { 4 => [1, 2, 3, 4, 5] } }
-    it_behaves_like 'a normalization algorithm'
-  end
-
-  # Runs standard Unicode normalization tests from `file_path` for a given `normalizer`. Expected invariants are
-  # specified via `invariants` hash.
-  #
-  # E.g., if `invariants` is { 2 => [1, 2, 3], 4 => [4, 5] } than the following invariants are expected to be true:
-  #
-  #   c2 == normalized(c1) == normalized(c2) == normalized(c3)
-  #   c4 == normalized(c4) == normalized(c5)
-  #
-  # where (c1, c2,...) are columns of the normalization test separated by semicolons and normalized() is the
-  # normalization function. Note, how expectation and tests columns indexes match the numbers in the `invariants` hash.
-  #
-  def run_test(normalizer, invariants, file_path)
-    File.open(file_path, 'r:UTF-8') do |file|
-      file.each do |line|
-        next if line.empty? || line =~ /^(@|#)/
-
-        data = line.split(';')[0...5].map { |cps| cps.split.map(&:hex) }
-
-        invariants.each do |expected_index, tests|
-          expected = data[expected_index - 1]
-
-          tests.each do |test_index|
-            test = data[test_index - 1]
-
-            normalized = normalizer.normalize_code_points(test)
-
-            message = error_message(line, test, expected, normalized, test_index, expected_index)
-            normalized.should(eq(expected), message)
-          end
-        end
-      end
-    end
-  end
-
-  # Generates a descriptive error message test failure.
-  #
-  def error_message(line, test, expected, normalized, test_index, expected_index)
-    <<-END
-Test:       "#{line.strip}"
-Invariant:  normalized(c#{test_index}) == c#{expected_index}
-Expected:   normalized(#{pretty_code_points(test).inspect}) == #{pretty_code_points(expected).inspect}
-Got:        #{pretty_code_points(normalized).inspect}
-    END
-  end
-
-  # Downloads full version of the test if necessary.
-  #
-  def prepare_full_test
-    return if File.file?(FULL_NORMALIZATION_TEST_PATH)
-
-    print '    Downloading NormalizationTest.txt ... '
-    File.open(FULL_NORMALIZATION_TEST_PATH, 'w') { |file| file.write(open(FULL_NORMALIZATION_TEST_URL).read) }
-    puts 'done.'
-  end
-
-  def pretty_code_points(code_points)
-    code_points.map { |cp| cp.to_s(16).rjust(4, '0').upcase }
-  end
-
-end

data/spec/tokenizers/base_spec.rb

@@ -1,259 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require File.join(File.dirname(File.dirname(__FILE__)), "spec_helper")
-include TwitterCldr::Tokenizers
-
-# normally, base can't be instantiated, so we have to patch it here
-module TwitterCldr
-  module Tokenizers
-    class Base
-      def init_resources; end
-    end
-  end
-end
-
-describe TwitterCldr::Tokenizers::Base do
-  before(:each) do
-    @base = TwitterCldr::Tokenizers::Base.new  # do NOT do this in production - must use subclass
-  end
-
-  it "should use English locale by default" do
-    @base.locale.should == :en
-  end
-
-  it "should use provided locale if there is one" do
-    TwitterCldr::Tokenizers::Base.new(:locale => :de).locale.should == :de
-  end
-
-  # tokenize_pattern is supposed to take a pattern found in the YAML resource files and break it into placeholders and plaintext.
-  # Placeholders are delimited by single and double curly braces, plaintext is everything else.
-  describe "#tokenize_pattern" do
-    context "with double curlies (path names, essentially)" do
-      it "should work with a placeholder only" do
-        @base.send(:tokenize_pattern, "{{place}}").should == [{ :value => "{{place}}", :type => :placeholder }]
-      end
-
-      it "should work with two placeholders separated by a space" do
-        @base.send(:tokenize_pattern, "{{first}} {{second}}").should == [{ :value => "{{first}}", :type => :placeholder },
-                                                                         { :value => " ", :type => :plaintext },
-                                                                         { :value => "{{second}}", :type => :placeholder }]
-      end
-
-      it "should work when surrounded by plaintext" do
-        @base.send(:tokenize_pattern, "being {{totally}} awesome").should == [{ :value => "being ", :type => :plaintext },
-                                                                              { :value => "{{totally}}", :type => :placeholder },
-                                                                              { :value => " awesome", :type => :plaintext }]
-      end
-    end
-
-    context "with single curlies (indexes)" do
-      it "should work with a placeholder only" do
-        @base.send(:tokenize_pattern, "{1}").should == [{ :value => "{1}", :type => :placeholder }]
-      end
-
-      it "should work with two placeholders separated by a space" do
-        @base.send(:tokenize_pattern, "{0} {1}").should == [{ :value => "{0}", :type => :placeholder },
-                                                            { :value => " ", :type => :plaintext },
-                                                            { :value => "{1}", :type => :placeholder }]
-      end
-
-      it "should work when surrounded by plaintext" do
-        @base.send(:tokenize_pattern, "only {1} dragon").should == [{ :value => "only ", :type => :plaintext },
-                                                                    { :value => "{1}", :type => :placeholder },
-                                                                    { :value => " dragon", :type => :plaintext }]
-      end
-    end
-  end
-
-  describe "#choose_placeholder" do
-    before(:each) do
-      @placeholders = [{ :name => "wallace", :object => "man" }, { :name => "gromit", :object => "dog" }]
-    end
-
-    it "should choose the correct named placeholder" do
-      @base.send(:choose_placeholder, "{{wallace}}", @placeholders).should == "man"
-      @base.send(:choose_placeholder, "{{gromit}}", @placeholders).should == "dog"
-    end
-
-    it "should choose the correct placeholder by array index" do
-      @base.send(:choose_placeholder, "{0}", @placeholders).should == "man"
-      @base.send(:choose_placeholder, "{1}", @placeholders).should == "dog"
-    end
-  end
-
-  describe "#expand_pattern" do
-    it "recursively calls expand_pattern if a symbol (keypath) is given" do
-      @base.type = :fake_type
-      mock(@base).traverse([:another, :path]) { "found_me" }
-      mock(@base).pattern_for("found_me") { "pattern_text" }
-      mock.proxy(@base).expand_pattern("pattern_text")
-      mock.proxy(@base).expand_pattern(:'another.path')
-      @base.send(:expand_pattern, :'another.path').should == [{ :value => "pattern_text", :type => :plaintext }]
-    end
-
-    it "expands placeholders as necessary" do
-      placeholder_obj = Object.new
-      mock(placeholder_obj).tokens(:type => :man) { ["token1", "token2"] }
-      @base.placeholders = [{ :name => "wallace", :object => placeholder_obj }]
-      @base.type = :man
-      @base.send(:expand_pattern, "{{wallace}} rules").should == ["token1", "token2", { :type => :plaintext, :value => " rules" }]
-    end
-
-    it "doesn't choke if the placeholder can't be found" do
-      @base.placeholders = [{ :name => "gromit", :object => "dog" }]
-      @base.type = :man
-      @base.send(:expand_pattern, "{{wallace}} rules").should == [{ :type => :plaintext, :value => " rules" }]
-    end
-  end
-
-  describe "#tokens_for" do
-    let(:token1) { Token.new(:value => "token1", :type => :plaintext) }
-    let(:token2) { Token.new(:value => "token2", :type => :plaintext) }
-
-    before(:each) do
-      stub(@base).traverse([:fake_key]) { "fake_pattern" }
-      stub(@base).pattern_for("fake_pattern") { "fake_expandable_pattern" }
-      stub(@base).expand_pattern("fake_expandable_pattern") { [token1, token2] }
-    end
-
-    it "caches tokens" do
-      @base.type = "fake_type"
-      result = @base.send(:tokens_for, [:fake_key])
-      result[0].value.should == "token1"
-      result[1].value.should == "token2"
-      @base.class.send(:class_variable_get, :'@@token_cache')["en|fake_key|fake_type|nil".hash].should == result
-
-      @base.type = "fake_type"
-      result_again = @base.send(:tokens_for, [:fake_key])
-      result_again.object_id.should == result.object_id
-    end
-
-    it "caches tokens per language" do
-      @base.type = "fake_type"
-      result_en = @base.send(:tokens_for, [:fake_key])
-      result_en[0].value.should == "token1"
-      result_en[1].value.should == "token2"
-      @base.class.send(:class_variable_get, :'@@token_cache')["en|fake_key|fake_type|nil".hash].should == result_en
-      result_en2 = @base.send(:tokens_for, [:fake_key])
-      result_en2.object_id.should == result_en.object_id
-
-      @base.instance_variable_set(:'@locale', :pt)
-      result_pt = @base.send(:tokens_for, [:fake_key])
-      result_pt[0].value.should == "token1"
-      result_pt[1].value.should == "token2"
-      @base.class.send(:class_variable_get, :'@@token_cache')["pt|fake_key|fake_type|nil".hash].should == result_pt
-      result_pt.object_id.should_not == result_en.object_id
-      result_pt2 = @base.send(:tokens_for, [:fake_key])
-      result_pt2.object_id.should == result_pt.object_id
-      result_pt2.object_id.should_not == result_en.object_id
-      result_pt2.object_id.should_not == result_en2.object_id
-    end
-  end
-
-  describe "#expand" do
-    it "leaves a normal hash alone" do
-      hash = { :twitter => { :is => "cool" } }
-      @base.send(:expand, hash, hash).should == hash
-    end
-
-    it "expands a single symbol path" do
-      hash = { :twitter => { :is => :'adjective.positive' }, :adjective => { :positive => "cool" } }
-      @base.send(:expand, hash, hash).should == { :twitter => { :is => "cool" }, :adjective => { :positive => "cool" } }
-    end
-
-    it "expands nested symbol paths" do
-      hash = { :twitter => { :is => :'adjective.positive' }, :adjective => { :positive => :'adjective.default', :default => "awesome" } }
-      @base.send(:expand, hash, hash).should == { :twitter => { :is => "awesome" }, :adjective => { :positive => "awesome", :default => "awesome" } }
-    end
-  end
-
-  describe "#traverse" do
-    let(:hash) { { :admiral => { :captain => { :commander => { :lieutenant => "Found Me!" } } } } }
-
-    it "finds the correct value in the hash" do
-      @base.send(:traverse, [:admiral, :captain, :commander, :lieutenant], hash).should == "Found Me!"
-    end
-
-    it "returns nil if the path doesn't exist" do
-      @base.send(:traverse, [:admiral, :captain, :commander, :lieutenant, :ensign], hash).should be_nil
-    end
-
-    it 'uses @resource if no hash is provided' do
-      old_resource = @base.instance_variable_get(:@resource)
-
-      begin
-        @base.instance_variable_set(:@resource, hash)
-        @base.send(:traverse, [:admiral, :captain, :commander, :lieutenant]).should == "Found Me!"
-      ensure
-        @base.instance_variable_set(:@resource, old_resource)
-      end
-    end
-  end
-
-  # Not to be confused with tokenize_pattern, which pulls out placeholders.  Tokenize_format actually splits a completely
-  # expanded format string into whatever parts are defined by the subclass's token type and token splitter regexes.
-  describe "#tokenize_format" do
-    it "assigns the right token types to the tokens" do
-      stub(@base).token_splitter_regexes do
-        { :else => /([abc])/ }
-      end
-
-      stub(@base).token_type_regexes do
-        {
-          :else => {
-            :a => { :regex => /a/, :priority => 1 },
-            :b => { :regex => /b/, :priority => 1 },
-            :c => { :regex => /c/, :priority => 1 },
-            :plaintext => { :regex => //, :priority => 2 }
-          }
-        }
-      end
-
-      tokens = @base.send(:tokenize_format, "a b c")
-      tokens.size.should == 5
-
-      tokens[0].value.should == "a"; tokens[0].type.should == :a
-      tokens[1].value.should == " "; tokens[1].type.should == :plaintext
-      tokens[2].value.should == "b"; tokens[2].type.should == :b
-      tokens[3].value.should == " "; tokens[3].type.should == :plaintext
-      tokens[4].value.should == "c"; tokens[4].type.should == :c
-    end
-  end
-
-  describe "#token_type_regexes_for" do
-    before(:each) do
-      @base.instance_variable_set(:'@token_type_regexes', {
-        :blarg => "blarg",
-        :else => "otherwise"
-      })
-    end
-
-    it "returns the correct regexes by name" do
-      @base.send(:token_type_regexes_for, :blarg).should == "blarg"
-    end
-
-    it "defaults to the regexes stored at :else if the name given can't be found" do
-      @base.send(:token_type_regexes_for, :foo).should == "otherwise"
-    end
-  end
-
-  describe "#token_splitter_regex_for" do
-    before(:each) do
-      @base.instance_variable_set(:'@token_splitter_regexes', {
-        :blarg => "blarg",
-        :else => "otherwise"
-      })
-    end
-
-    it "returns the correct regexes by name" do
-      @base.send(:token_splitter_regex_for, :blarg).should == "blarg"
-    end
-
-    it "defaults to the regexes stored at :else if the name given can't be found" do
-      @base.send(:token_splitter_regex_for, :foo).should == "otherwise"
-    end
-  end
-end

data/spec/utils/territories_spec.rb

@@ -1,16 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2012 Twitter, Inc
-# http://www.apache.org/licenses/LICENSE-2.0
-
-require 'spec_helper'
-
-describe TwitterCldr::Utils::Territories do
-  describe "#deep_normalize_territory_code_keys" do
-    TwitterCldr::Utils::Territories.deep_normalize_territory_code_keys(
-      { "is" => [ { "US" => "United States",
-                    5 => "Suður-Ameríka" },
-                  { "009" => "Eyjaálfa" } ] }
-    ).should == { :is => [ { :us => "United States" }, { } ] }
-  end
-end