twitter_cldr 3.2.1 → 3.3.0

data/spec/shared/postal_codes_spec.rb

@@ -78,13 +78,33 @@ describe PostalCodes do
         postal_code = PostalCodes.for_territory(territory)
 
         it "returns samples that match #{territory}" do
-          postal_code.sample(10).each do |sample|
-            result = postal_code.valid?(sample)
-            puts "Failed with example #{sample}" unless result
-            result.should be_true
+          if postal_code.has_generator?
+            postal_code.sample(10).each do |sample|
+              result = postal_code.valid?(sample)
+              puts "Failed with example #{sample}" unless result
+              result.should be_true
+            end
           end
         end
       end
     end
   end
+
+  context 'with a postal code that has no AST' do
+    let(:postal_code) { PostalCodes.new(:xx, /\d{5}/, nil) }
+
+    describe '#sample' do
+      it 'raises an exception' do
+        expect { postal_code.sample }.to raise_error(
+          MissingPostcodeGeneratorError
+        )
+      end
+    end
+
+    describe '#has_generator?' do
+      it 'returns false' do
+        expect(postal_code).to_not have_generator
+      end
+    end
+  end
 end

data/spec/shared/properties_database_spec.rb

@@ -0,0 +1,157 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+require 'fileutils'
+require 'securerandom'
+require 'tmpdir'
+
+include TwitterCldr::Shared
+include TwitterCldr::Utils
+
+describe PropertiesDatabase do
+  context 'with an empty database' do
+    let(:tmp_dir) do
+      File.join(Dir.tmpdir, SecureRandom.hex)
+    end
+
+    let(:database) { PropertiesDatabase.new(tmp_dir) }
+
+    before(:each) do
+      FileUtils.mkdir_p(tmp_dir)
+    end
+
+    after(:each) do
+      FileUtils.rm_rf(tmp_dir)
+    end
+
+    describe '#store' do
+      it 'associates the code points with the property name/value' do
+        database.store('foo', 'bar', RangeSet.new([1..4]))
+        result = database.code_points_for_property('foo', 'bar')
+        expect(result).to be_a(RangeSet)
+        expect(result.to_a).to eq([1..4])
+      end
+    end
+  end
+
+  context 'with a full database of properties' do
+    let(:database) { PropertiesDatabase.new }
+
+    describe '#code_points_for_property' do
+      it 'retrieves code points for the property name' do
+        code_points = database.code_points_for_property('Math')
+        [126, 172, 215, 1014, 9168, 10176].each do |cp|
+          expect(code_points).to include(cp)
+        end
+      end
+
+      it 'retrieves code points for the property name/value pair' do
+        code_points = database.code_points_for_property('Age', '1.1')
+        [501, 736, 890, 990, 1118, 1227, 1632].each do |cp|
+          expect(code_points).to include(cp)
+        end
+      end
+
+      it "returns an empty range set if the property name can't be found" do
+        code_points = database.code_points_for_property('foo')
+        expect(code_points.to_a).to eq([])
+      end
+
+      it "returns an empty range set if the property name/value pair can't be found" do
+        code_points = database.code_points_for_property('Age', 'foo')
+        expect(code_points.to_a).to eq([])
+      end
+    end
+
+    describe '#include?' do
+      it 'returns true if the database contains the property name' do
+        expect(database.include?('Age')).to be_true
+      end
+
+      it 'returns true if the database contains the property name/value pair' do
+        expect(database.include?('Age', '1.1')).to be_true
+      end
+
+      it "returns false if the database doesn't contain the property name" do
+        expect(database.include?('foo')).to be_false
+      end
+
+      it "returns false if the database doesn't contain the property name/value pair" do
+        expect(database.include?('Age', 'foo')).to be_false
+      end
+    end
+
+    describe '#properties_for_code_point' do
+      it 'returns a property set for the given code point' do
+        property_set = database.properties_for_code_point(65)
+        expect(property_set).to be_a(PropertySet)
+        expect(property_set.general_category).to eq(Set.new(%w(L Lu)))
+        expect(property_set.word_break).to eq(Set.new(%w(ALetter)))
+      end
+    end
+
+    describe '#property_names' do
+      it 'returns a list of all valid property names' do
+        %w(Radical Script Hex_Digit Diacritic).each do |property_name|
+          expect(database.property_names).to include(property_name)
+        end
+      end
+    end
+
+    describe '#property_values_for' do
+      it 'returns a list of all valid values for a property name' do
+        values = database.property_values_for('Script')
+        %w(Cyrillic Latin Hangul Han Bengali Tagalog Yi).each do |script|
+          expect(values).to include(script)
+        end
+      end
+
+      it 'correctly expands value prefixes for General_Category' do
+        values = database.property_values_for('General_Category')
+        expect(values).to include('Cc')
+        expect(values).to include('C')
+
+        expect(values).to include('Lo')
+        expect(values).to include('L')
+
+        expect(values).to include('Pd')
+        expect(values).to include('P')
+
+        expect(values).to include('Sm')
+        expect(values).to include('S')
+      end
+    end
+
+    describe '#each_property_pair' do
+      it 'yields each property name/value pair' do
+        pairs = database.each_property_pair.to_a
+
+        expected_pairs = [
+          %w(Script Thai), ['Radical', nil],
+          ['Math', nil], %w(Line_Break HY),
+          %w(Jamo_Short_Name EU), ['Hyphen', nil]
+        ]
+
+        expected_pairs.each do |expected_pair|
+          expect(pairs).to include(expected_pair)
+        end
+      end
+    end
+
+    describe '#normalize' do
+      let(:property_name) { 'foo' }
+      let(:property_value) { 'bar' }
+
+      it 'delegates normalization to a normalizer' do
+        normalizer = Object.new
+        stub(database).normalizer { normalizer }
+        mock(normalizer).normalize(property_name, property_value)
+        database.normalize(property_name, property_value)
+      end
+    end
+  end
+end

data/spec/shared/property_name_aliases_spec.rb

@@ -0,0 +1,56 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe PropertyNameAliases do
+  let(:alias_class) { PropertyNameAliases }
+
+  describe '#abbreviated_alias_for' do
+    it 'finds the abbreviated alias for a long property name' do
+      expect(alias_class.abbreviated_alias_for('Script')).to eq('sc')
+      expect(alias_class.abbreviated_alias_for('Age')).to eq('age')
+      expect(alias_class.abbreviated_alias_for('Uppercase_Mapping')).to eq('uc')
+    end
+
+    it 'returns nil if no alias can be found' do
+      expect(alias_class.abbreviated_alias_for('foo')).to be_nil
+    end
+  end
+
+  describe '#long_alias_for' do
+    it 'finds the long alias for an abbreviated property name' do
+      expect(alias_class.long_alias_for('sc')).to eq('Script')
+      expect(alias_class.long_alias_for('age')).to eq('Age')
+      expect(alias_class.long_alias_for('uc')).to eq('Uppercase_Mapping')
+    end
+
+    it 'returns nil if no alias can be found' do
+      expect(alias_class.long_alias_for('foo')).to be_nil
+    end
+  end
+
+  describe '#aliases_for' do
+    examples = {
+      'cjkRSUnicode' => %w(kRSUnicode Unicode_Radical_Stroke URS),
+      'WSpace' => %w(White_Space space),
+      'scf' => %w(Simple_Case_Folding sfc),
+      'IDS' => %w(ID_Start)
+    }
+
+    examples.each_pair do |property_name, aliases|
+      it "finds all the known aliases for the #{property_name} property name" do
+        found = alias_class.aliases_for(property_name)
+        expect(found).to match_array(aliases)
+      end
+    end
+
+    it 'returns an empty array if no aliases are found' do
+      expect(alias_class.aliases_for('foo')).to eq([])
+    end
+  end
+end

data/spec/shared/property_normalizer_spec.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe PropertyNormalizer do
+  let(:database) { PropertiesDatabase.new }
+  let(:normalizer) { PropertyNormalizer.new(database) }
+
+  describe '#normalize' do
+    it 'correctly normalizes just property names' do
+      name, value = normalizer.normalize('IDS')
+      expect(name).to eq('ID_Start')
+      expect(value).to be_nil
+    end
+
+    it 'correctly normalizes property names and values' do
+      %w(age Age).each do |name|
+        %w(1.1 V1_1).each do |value|
+          cur_name, cur_value = normalizer.normalize(name, value)
+          expect(cur_name).to eq('Age')
+          expect(cur_value).to eq('1.1')
+        end
+      end
+    end
+
+    it 'correctly normalizes properties that support numeric values' do
+      %w(ccc Canonical_Combining_Class).each do |name|
+        %w(0 NR Not_Reordered).each do |value|
+          cur_name, cur_value = normalizer.normalize(name, value)
+          expect(cur_name).to eq('Canonical_Combining_Class')
+          expect(cur_value).to eq('0')
+        end
+      end
+    end
+
+    it "returns a blank property when the name and value can't be found" do
+      name, value = normalizer.normalize('foo', 'bar')
+      expect(name).to be_nil
+      expect(value).to be_nil
+    end
+
+    it 'handles valid name but invalid value' do
+      name, value = normalizer.normalize('ideographic', 'ideographic')
+      expect(name).to be_nil
+      expect(value).to be_nil
+    end
+
+    it 'fixes casing of property names' do
+      name, value = normalizer.normalize('ideographic')
+      expect(name).to eq('Ideographic')
+    end
+
+    it 'fixes casing of property names and values' do
+      name, value = normalizer.normalize('sc', 'han')
+      expect(name).to eq('Script')
+      expect(value).to eq('Han')
+    end
+  end
+end

data/spec/shared/property_set_spec.rb

@@ -0,0 +1,218 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe PropertySet do
+  let(:properties_hash) { {} }
+  let(:property_set) { PropertySet.new(properties_hash) }
+
+  describe '#age' do
+    it 'defaults to "Unassigned"' do
+      expect(property_set.age).to eq(['Unassigned'])
+    end
+
+    it 'returns the age when one is supplied' do
+      properties_hash.merge!('Age' => ['foo'])
+      expect(property_set.age).to eq(['foo'])
+    end
+  end
+
+  describe '#joining_type' do
+    it 'defaults to "Non_Joining"' do
+      expect(property_set.joining_type).to eq(['Non_Joining'])
+    end
+
+    it 'defaults to the joining type that corresponds to the general category (if set)' do
+      properties_hash.merge!('General_Category' => ['Mn'])
+      expect(property_set.joining_type).to eq(['Transparent'])
+    end
+
+    it 'returns the joining type when one is supplied' do
+      properties_hash.merge!('Joining_Type' => ['foo'])
+      expect(property_set.joining_type).to eq(['foo'])
+    end
+  end
+
+  describe '#bidi_paired_bracket_type' do
+    it 'defaults to "None"' do
+      expect(property_set.bidi_paired_bracket_type).to eq(['None'])
+    end
+
+    it 'returns the bracket type when one is supplied' do
+      properties_hash.merge!('Bidi_Paired_Bracket_Type' => ['foo'])
+      expect(property_set.bidi_paired_bracket_type).to eq(['foo'])
+    end
+  end
+
+  describe '#block' do
+    it 'defaults to "No_Block"' do
+      expect(property_set.block).to eq(['No_Block'])
+    end
+
+    it 'returns the block when one is supplied' do
+      properties_hash.merge!('Block' => ['foo'])
+      expect(property_set.block).to eq(['foo'])
+    end
+  end
+
+  describe '#east_asian_width' do
+    it 'defaults to "N"' do
+      expect(property_set.east_asian_width).to eq(['N'])
+    end
+
+    it 'returns the width when one is supplied' do
+      properties_hash.merge!('East_Asian_Width' => ['foo'])
+      expect(property_set.east_asian_width).to eq(['foo'])
+    end
+  end
+
+  describe '#grapheme_cluster_break' do
+    it 'defaults to "Other"' do
+      expect(property_set.grapheme_cluster_break).to eq(['Other'])
+    end
+
+    it 'returns the break when one is supplied' do
+      properties_hash.merge!('Grapheme_Cluster_Break' => ['foo'])
+      expect(property_set.grapheme_cluster_break).to eq(['foo'])
+    end
+  end
+
+  describe '#hangul_syllable_type' do
+    it 'defaults to "Not_Applicable"' do
+      expect(property_set.hangul_syllable_type).to eq(['Not_Applicable'])
+    end
+
+    it 'returns the syllable type when one is supplied' do
+      properties_hash.merge!('Hangul_Syllable_Type' => ['foo'])
+      expect(property_set.hangul_syllable_type).to eq(['foo'])
+    end
+  end
+
+  describe '#indic_positional_category' do
+    it 'defaults to "NA"' do
+      expect(property_set.indic_positional_category).to eq(['NA'])
+    end
+
+    it 'returns the positional category if one is supplied' do
+      properties_hash.merge!('Indic_Positional_Category' => ['foo'])
+      expect(property_set.indic_positional_category).to eq(['foo'])
+    end
+  end
+
+  describe '#indic_syllabic_category' do
+    it 'defaults to "Other"' do
+      expect(property_set.indic_syllabic_category).to eq(['Other'])
+    end
+
+    it 'returns the syllabic category if one is supplied' do
+      properties_hash.merge!('Indic_Syllabic_Category' => ['foo'])
+      expect(property_set.indic_syllabic_category).to eq(['foo'])
+    end
+  end
+
+  describe '#jamo_short_name' do
+    it 'defaults to "<none>"' do
+      expect(property_set.jamo_short_name).to eq(['<none>'])
+    end
+
+    it 'returns the short name if one is supplied' do
+      properties_hash.merge!('Jamo_Short_Name' => ['foo'])
+      expect(property_set.jamo_short_name).to eq(['foo'])
+    end
+  end
+
+  describe '#line_break' do
+    it 'defaults to XX' do
+      expect(property_set.line_break).to eq(['XX'])
+    end
+
+    it 'returns the line break category if one is supplied' do
+      properties_hash.merge!('Line_Break' => ['foo'])
+      expect(property_set.line_break).to eq(['foo'])
+    end
+  end
+
+  describe '#general_category' do
+    it 'defaults to nil' do
+      expect(property_set.general_category).to eq([])
+    end
+
+    it 'returns the general category if one is supplied' do
+      properties_hash.merge!('General_Category' => ['foo'])
+      expect(property_set.general_category).to eq(['foo'])
+    end
+  end
+
+  describe '#script_extensions' do
+    it 'defaults to "<script>"' do
+      expect(property_set.script_extensions).to eq(['<script>'])
+    end
+
+    it 'returns the script extension if one is supplied' do
+      properties_hash.merge!('Script_Extensions' => ['foo'])
+      expect(property_set.script_extensions).to eq(['foo'])
+    end
+  end
+
+  describe '#script' do
+    it 'defaults to "Unknown"' do
+      expect(property_set.script).to eq(['Unknown'])
+    end
+
+    it 'returns the script if one is supplied' do
+      properties_hash.merge!('Script' => ['foo'])
+      expect(property_set.script).to eq(['foo'])
+    end
+  end
+
+  describe '#sentence_break' do
+    it 'defaults to "Other"' do
+      expect(property_set.sentence_break).to eq(['Other'])
+    end
+
+    it 'returns the sentence break category if one is supplied' do
+      properties_hash.merge!('Sentence_Break' => ['foo'])
+      expect(property_set.sentence_break).to eq(['foo'])
+    end
+  end
+
+  describe '#word_break' do
+    it 'defaults to "Other"' do
+      expect(property_set.word_break).to eq(['Other'])
+    end
+
+    it 'returns the word break category if one is supplied' do
+      properties_hash.merge!('Word_Break' => ['foo'])
+      expect(property_set.word_break).to eq(['foo'])
+    end
+  end
+
+  describe 'properties without defaults' do
+    CodePoint.properties.property_names.each do |property_name|
+      method_name = property_name.downcase
+
+      if PropertySet::AdditionalPropertyMethods.method_defined?(method_name)
+        describe "##{method_name}" do
+          it 'returns true if the property has been set but has no value' do
+            properties_hash.merge!(property_name => nil)
+            expect(property_set.send(method_name)).to eq(true)
+          end
+
+          it 'returns false if the property has not been set' do
+            expect(property_set.send(method_name)).to eq(false)
+          end
+
+          it 'returns the property value if it has been set' do
+            properties_hash.merge!(property_name => ['foo'])
+            expect(property_set.send(method_name)).to eq(['foo'])
+          end
+        end
+      end
+    end
+  end
+end

data/spec/shared/property_value_aliases_spec.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe PropertyValueAliases do
+  let(:alias_class) { PropertyValueAliases }
+
+  describe '#abbreviated_alias_for' do
+    it 'finds the abbreviated alias for a long property value' do
+      expect(alias_class.abbreviated_alias_for('AHex', 'No')).to eq('N')
+      expect(alias_class.abbreviated_alias_for('AHex', 'Yes')).to eq('Y')
+
+      expect(alias_class.abbreviated_alias_for('dt', 'Canonical')).to eq('Can')
+      expect(alias_class.abbreviated_alias_for('dt', 'Circle')).to eq('Enc')
+      expect(alias_class.abbreviated_alias_for('dt', 'Final')).to eq('Fin')
+    end
+
+    it 'returns nil if no alias can be found' do
+      expect(alias_class.abbreviated_alias_for('AHex', 'foo')).to be_nil
+    end
+  end
+
+  describe '#long_alias_for' do
+    it 'finds the long alias for an abbreviated property value' do
+      expect(alias_class.long_alias_for('AHex', 'N')).to eq('No')
+      expect(alias_class.long_alias_for('AHex', 'Y')).to eq('Yes')
+
+      expect(alias_class.long_alias_for('dt', 'Can')).to eq('Canonical')
+      expect(alias_class.long_alias_for('dt', 'Enc')).to eq('Circle')
+      expect(alias_class.long_alias_for('dt', 'Fin')).to eq('Final')
+    end
+
+    it 'returns nil if no alias can be found' do
+      expect(alias_class.long_alias_for('AHex', 'foo')).to be_nil
+    end
+  end
+
+  describe '#numeric_alias_for' do
+    it 'finds the numeric alias (if one exists) for a long or abbreviated property value' do
+      %w(0 NR Not_Reordered).each do |value|
+        expect(alias_class.numeric_alias_for('ccc', value)).to eq('0')
+      end
+    end
+  end
+
+  describe '#aliases_for' do
+    it 'compiles a list of value aliases for the given property name' do
+      expect(alias_class.aliases_for('ccc', 'NR')).to match_array(%w(0 Not_Reordered))
+      expect(alias_class.aliases_for('ccc', 'Not_Reordered')).to match_array(%w(0 NR))
+      expect(alias_class.aliases_for('ccc', '0')).to match_array(%w(NR Not_Reordered))
+    end
+  end
+end

data/spec/shared/territories_spec.rb

@@ -121,7 +121,7 @@ describe Territories do
       { "is" => [ { "US" => "United States",
                     5 => "Suður-Ameríka" },
                   { "009" => "Eyjaálfa" } ] }
-    ).should == { :is => [ { :us => "United States" }, { } ] }
+    ).should == { is: [ { us: "United States" }, { } ] }
   end
 
 end

data/spec/shared/unicode_regex_spec.rb

@@ -60,6 +60,18 @@ describe UnicodeRegex do
           "(?:[\\u{0067}-\\u{006b}])(?:[\\u{0070}-\\u{0073}])"
         )
       end
+
+      it "supports modifiers" do
+        regex = UnicodeRegex.compile('abc', 'm').to_regexp
+        expect(regex.options).to eq(Regexp::MULTILINE)
+      end
+
+      it "supports multiple modifiers at once" do
+        regex = UnicodeRegex.compile('abc', 'mi').to_regexp
+        expect(regex.options).to eq(
+          Regexp::MULTILINE | Regexp::IGNORECASE
+        )
+      end
     end
   end
 
@@ -160,6 +172,20 @@ describe UnicodeRegex do
         expect(regex).not_to exactly_match("a")
       end
 
+      it "should match a regex containing a unioned character set" do
+        regex = compile("[[:L:][:White_Space:]]*")
+        expect(regex).to exactly_match("abc")
+        expect(regex).to exactly_match("くøß")
+        expect("a b c _ d".gsub(regex.to_regexp, "")).to eq("_")
+      end
+
+      it "should match a regex containing a negated unioned character set" do
+        regex = compile("[^[:L:][:White_Space:]]*")
+        expect(regex).to exactly_match(".,/")
+        expect(regex).to_not exactly_match("a b c")
+        expect("a b c _ d".gsub(regex.to_regexp, "")).to eq("a b c  d")
+      end
+
       it "should match a regex containing a negated character set (alternate syntax)" do
         regex = compile("[[:^Zs:]]")
         expect(regex).to exactly_match("a")
@@ -192,12 +218,19 @@ describe UnicodeRegex do
       end
 
       it "should match a regex with a complicated expression inside the char class" do
-        # not [separators U space-tilde] diff [letters diff numbers]  (diff is commutative)
-        regex = compile("[^[\\p{Z}\\u0020-\\u007f]-[\\p{L}]-[\\p{N}]]")
+        # [separators U space-tilde] diff [letters diff numbers]  (diff is commutative)
+        regex = compile("[[\\p{Z}\\u0020-\\u007f]-[\\p{L}]-[\\p{N}]]")
         expect(regex).to exactly_match(" ")
         expect(regex).to exactly_match(",")
         expect(regex).not_to exactly_match("a")
       end
+
+      it "should treat a dash that is the first character of a character class as a literal dash instead of a range" do
+        regex = compile("[-abc]*")
+        expect(regex).to exactly_match("a-b-c")
+        expect(regex).to exactly_match("--a")
+        expect(regex).not_to exactly_match("def")
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -48,9 +48,9 @@ end
 RSpec.configure do |config|
   config.mock_with :rr
 
-  config.filter_run(:focus => true)
+  config.filter_run(focus: true)
   config.run_all_when_everything_filtered = true
-  config.filter_run_excluding(:slow => true) unless ENV['FULL_SPEC']
+  config.filter_run_excluding(slow: true) unless ENV['FULL_SPEC']
 
   config.before(:each) do
     TwitterCldr.reset_locale_fallbacks
@@ -62,7 +62,7 @@ end
 
 RSpec::Matchers.define :match_normalized do |expected|
   match do |actual|
-    expected.localize.normalize(:using => :NFKC).to_s == actual.localize.normalize(:using => :NFKC).to_s
+    expected.localize.normalize(using: :NFKC).to_s == actual.localize.normalize(using: :NFKC).to_s
   end
 end