twitter_cldr 3.2.1 → 3.3.0

data/spec/{shared → segmentation}/break_iterator_spec.rb

@@ -5,11 +5,11 @@
 
 require 'spec_helper'
 
-include TwitterCldr::Shared
+include TwitterCldr::Segmentation
 
 describe BreakIterator do
   describe "#each_sentence" do
-    let(:iterator) { BreakIterator.new(:en, :use_uli_exceptions => true) }
+    let(:iterator) { BreakIterator.new(:en, use_uli_exceptions: true) }
 
     it "should return an enumerator if called without a block" do
       expect(iterator.each_sentence("foo bar")).to be_a(Enumerator)
@@ -18,31 +18,31 @@ describe BreakIterator do
     it "splits a simple string into sentences" do
       str = "The. Quick. Brown. Fox."
       expect(iterator.each_sentence(str).to_a).to eq([
-        "The.", " Quick.", " Brown.", " Fox."
+        "The. ", "Quick. ", "Brown. ", "Fox."
       ])
     end
 
     it "does not split on commas, for example" do
       str = "The. Quick, brown. Fox."
       expect(iterator.each_sentence(str).to_a).to eq([
-        "The.", " Quick, brown.", " Fox."
+        "The. ", "Quick, brown. ", "Fox."
       ])
     end
 
     it "does not split periods in the midst of other letters, eg. in a URL" do
       str = "Visit us. Go to http://translate.twitter.com."
       expect(iterator.each_sentence(str).to_a).to eq([
-        "Visit us.",
-        " Go to http://translate.twitter.com."
+        "Visit us. ",
+        "Go to http://translate.twitter.com."
       ])
     end
 
     it "splits on sentences that end with other kinds of punctuation" do
       str = "Help us translate! Speak another language? You really, really rock."
       expect(iterator.each_sentence(str).to_a).to eq([
-        "Help us translate!",
-        " Speak another language?",
-        " You really, really rock."
+        "Help us translate! ",
+        "Speak another language? ",
+        "You really, really rock."
       ])
     end
 
@@ -50,23 +50,52 @@ describe BreakIterator do
       it "does not split on certain abbreviations like Mr. and Mrs." do
         str = "I really like Mrs. Patterson. She's nice."
         expect(iterator.each_sentence(str).to_a).to eq([
-          "I really like Mrs. Patterson.",
-          " She's nice."
+          "I really like Mrs. Patterson. ",
+          "She's nice."
         ])
       end
     end
 
     context "without ULI exceptions" do
-      let(:iterator) { BreakIterator.new(:en, :use_uli_exceptions => false) }
+      let(:iterator) { BreakIterator.new(:en, use_uli_exceptions: false) }
 
       it "splits on certain abbreviations like Mr. and Mrs. (use ULI rules to avoid this behavior)" do
         str = "I really like Mrs. Patterson. She's nice."
         expect(iterator.each_sentence(str).to_a).to eq([
-          "I really like Mrs.",
-          " Patterson.",
-          " She's nice."
+          "I really like Mrs. ",
+          "Patterson. ",
+          "She's nice."
         ])
       end
     end
   end
-end
+
+  describe "#each_word" do
+    let(:iterator) { BreakIterator.new(:en) }
+
+    it "should return an enumerator if called without a block" do
+      expect(iterator.each_word("foo bar")).to be_a(Enumerator)
+    end
+
+    it "splits a simple string into words" do
+      str = "the quick brown fox"
+      expect(iterator.each_word(str).to_a).to eq([
+        "the", " ", "quick", " ", "brown", " ", "fox"
+      ])
+    end
+
+    it "breaks around periods" do
+      str = "The. Quick. Brown. Fox."
+      expect(iterator.each_word(str).to_a).to eq([
+        "The", ".", " ", "Quick", ".", " ", "Brown", ".", " ", "Fox", "."
+      ])
+    end
+
+    it "does not break at apostrophes" do
+      str = "I like cats. They're cute."
+      expect(iterator.each_word(str).to_a).to eq([
+        "I", " ", "like", " ", "cats", ".", " ", "They're", " ", "cute", "."
+      ])
+    end
+  end
+end

data/spec/segmentation/parser_spec.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr
+include TwitterCldr::Parsers
+
+describe 'Segmentation' do
+  let(:parser) { Segmentation::Parser.new }
+
+  def parse(tokens, options = {})
+    parser.parse(tokens, options)
+  end
+
+  describe Segmentation::Parser do
+    let(:symbol_table) do
+      SymbolTable.new({
+        "$FOO" => parser.tokenize_regex("[abc]")
+      })
+    end
+
+    describe "#parse" do
+      it "should parse a rule with a break" do
+        rule = parse("[a-z] ÷ [0-9]")
+        expect(rule.left.to_regexp_str).to eq("(?:[\\u{0061}-\\u{007a}])")
+        expect(rule.right.to_regexp_str).to eq("(?:[\\u{0030}-\\u{0039}])")
+        expect(rule.boundary_symbol).to eq(:break)
+      end
+
+      it "should parse a rule with a non-break" do
+        rule = parse("[a-z] × [0-9]")
+
+        expect(rule.left.to_regexp_str).to eq(
+          "(?:[\\u{0061}-\\u{007a}])"
+        )
+
+        expect(rule.right.to_regexp_str).to eq(
+          "(?:[\\u{0030}-\\u{0039}])"
+        )
+
+        expect(rule.boundary_symbol).to eq(:no_break)
+      end
+
+      it "should parse a rule containing a variable" do
+        rule = parse("$FOO × bar", symbol_table: symbol_table)
+
+        expect(rule.left.to_regexp_str).to eq(
+          "(?:[\\u{0061}-\\u{0063}])"
+        )
+
+        expect(rule.right.to_regexp_str).to eq(
+          "(?:\\u{0062})(?:\\u{0061})(?:\\u{0072})"
+        )
+
+        expect(rule.boundary_symbol).to eq(:no_break)
+      end
+    end
+  end
+
+  describe Segmentation::BreakRule do
+    describe "#match" do
+      let(:rule) { parse("[a-z] ÷ [0-9]") }
+
+      it "rule should be the right type" do
+        expect(rule).to be_a(Segmentation::BreakRule)
+      end
+
+      it "should match and return the right offset and text" do
+        cursor = Segmentation::Cursor.new("c7")
+        match = rule.match(cursor)
+        expect(match.boundary_offset).to eq([0, 2])
+        expect(match.boundary_position).to eq(1)
+      end
+
+      it "should not match if the input string doesn't contain a matching right- and/or left-hand side" do
+        expect(rule.match(Segmentation::Cursor.new("C7"))).to be_nil
+        expect(rule.match(Segmentation::Cursor.new("cc"))).to be_nil
+        expect(rule.match(Segmentation::Cursor.new("CC"))).to be_nil
+      end
+    end
+  end
+
+  describe Segmentation::NoBreakRule do
+    describe "#match" do
+      let(:rule) { parse("[a-z] × [0-9]") }
+
+      it "rule should be the right type" do
+        expect(rule).to be_a(Segmentation::NoBreakRule)
+      end
+
+      it "should match and return the right offset and text" do
+        match = rule.match(Segmentation::Cursor.new("c7"))
+        expect(match.boundary_offset).to eq([0, 2])
+        expect(match.boundary_position).to eq(1)
+      end
+
+      it "should not match if the input string doesn't contain matching text" do
+        expect(rule.match(Segmentation::Cursor.new("C7"))).to be_nil
+        expect(rule.match(Segmentation::Cursor.new("cc"))).to be_nil
+        expect(rule.match(Segmentation::Cursor.new("CC"))).to be_nil
+      end
+    end
+  end
+end

data/spec/segmentation/rule_set_spec.rb

@@ -0,0 +1,102 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr
+
+describe Segmentation::RuleSet do
+  let(:test_path) do
+    File.join(
+      TwitterCldr::RESOURCES_DIR, 'shared', 'segments', 'tests'
+    )
+  end
+
+  def parse(test_data)
+    parts = test_data
+      .split(/([÷×])/)
+      .map(&:strip)
+      .reject(&:empty?)
+      .map do |part|
+        if part =~ /[÷×]/
+          part
+        else
+          [part.to_i(16)].pack('U*')
+        end
+      end
+  end
+
+  def boundaries(test_parts)
+    idx = 0
+
+    [].tap do |boundaries|
+      test_parts.each do |part|
+        if part =~ /[÷×]/
+          boundaries << idx if part == '÷'
+        else
+          idx += 1
+        end
+      end
+    end
+  end
+
+  def string(test_parts)
+    test_parts.select.with_index do |part, idx|
+      idx % 2 == 1
+    end.join
+  end
+
+  def error_message(test, test_case_boundaries, result_boundaries)
+<<END
+Expected boundaries to match test case
+
+            test case: #{test}
+  expected boundaries: #{test_case_boundaries.inspect}
+    actual boundaries: #{result_boundaries.inspect}
+END
+  end
+
+  shared_examples 'a conformant implementation' do
+    it 'passes all Unicode test cases' do
+      test_data.each_with_index do |test, idx|
+        next if skip_cases.include?(test)
+        test_parts = parse(test)
+        test_case_boundaries = boundaries(test_parts)
+        test_case_string = string(test_parts)
+        result_boundaries = rule_set.each_boundary(test_case_string).to_a
+        expect(result_boundaries).to(
+          eq(test_case_boundaries), error_message(
+            test, test_case_boundaries, result_boundaries
+          )
+        )
+      end
+    end
+  end
+
+  describe 'word boundaries' do
+    let(:test_file) { File.join(test_path, 'word_break_test.yml') }
+    let(:test_data) { YAML.load_file(test_file) }
+    let(:rule_set) { Segmentation::RuleSet.load(:en, 'word') }
+
+    # These cases don't work because they end in single quotes (0027).
+    # Conformant implementations (eg ICU) seem to allow partial regex
+    # matching, or allow matches to run off the end of the string.
+    # Since there's no such thing as a partial regex match in Ruby,
+    # we have to ignore these cases. Hopefully they happen infrequently
+    # in practice.
+    let(:skip_cases) { ['÷ 05D0 × 0027 ÷', '÷ 05D0 × 0308 × 0027 ÷'] }
+
+    it_behaves_like 'a conformant implementation'
+  end
+
+  describe 'sentence boundaries' do
+    let(:test_file) { File.join(test_path, 'sentence_break_test.yml') }
+    let(:test_data) { YAML.load_file(test_file) }
+    let(:rule_set) { Segmentation::RuleSet.load(:en, 'sentence') }
+    let(:skip_cases) { [] }
+
+    it_behaves_like 'a conformant implementation'
+  end
+end

data/spec/shared/calendar_spec.rb

@@ -68,12 +68,12 @@ describe Calendar do
 
     context 'when some data is missing' do
       it 'returns nil if some names format is missing' do
-        stub(TwitterCldr).get_locale_resource { { :de => { :calendars => { :gregorian => { :months => { :'stand-alone' => {} } } } } } }
+        stub(TwitterCldr).get_locale_resource { { de: { calendars: { gregorian: { months: { :'stand-alone' => {} } } } } } }
         expect(calendar.months(:wide)).to eq(nil)
       end
 
       it 'returns nil if calendars data is missing' do
-        stub(TwitterCldr).get_locale_resource { { :de => {} } }
+        stub(TwitterCldr).get_locale_resource { { de: {} } }
         expect(calendar.months(:wide)).to eq(nil)
       end
     end
@@ -83,41 +83,41 @@ describe Calendar do
     context 'when data is available' do
       it 'returns weekdays list in a wide names form by default' do
         expect(calendar.weekdays).to eq({
-            :sun => 'Sonntag',
-            :mon => 'Montag',
-            :tue => 'Dienstag',
-            :wed => 'Mittwoch',
-            :thu => 'Donnerstag',
-            :fri => 'Freitag',
-            :sat => 'Samstag'
+            sun: 'Sonntag',
+            mon: 'Montag',
+            tue: 'Dienstag',
+            wed: 'Mittwoch',
+            thu: 'Donnerstag',
+            fri: 'Freitag',
+            sat: 'Samstag'
         })
       end
 
       it 'supports wide names form' do
         expect(calendar.weekdays(:wide)).to eq({
-            :sun => 'Sonntag',
-            :mon => 'Montag',
-            :tue => 'Dienstag',
-            :wed => 'Mittwoch',
-            :thu => 'Donnerstag',
-            :fri => 'Freitag',
-            :sat => 'Samstag'
+            sun: 'Sonntag',
+            mon: 'Montag',
+            tue: 'Dienstag',
+            wed: 'Mittwoch',
+            thu: 'Donnerstag',
+            fri: 'Freitag',
+            sat: 'Samstag'
         })
       end
 
       it 'supports narrow names form' do
-        expect(calendar.weekdays(:narrow)).to eq({ :sun => 'S', :mon => 'M', :tue => 'D', :wed => 'M', :thu => 'D', :fri => 'F', :sat => 'S' })
+        expect(calendar.weekdays(:narrow)).to eq({ sun: 'S', mon: 'M', tue: 'D', wed: 'M', thu: 'D', fri: 'F', sat: 'S' })
       end
 
       it 'supports abbreviated names form' do
         expect(calendar.weekdays(:abbreviated)).to eq({
-            :sun => 'So',
-            :mon => 'Mo',
-            :tue => 'Di',
-            :wed => 'Mi',
-            :thu => 'Do',
-            :fri => 'Fr',
-            :sat => 'Sa'
+            sun: 'So',
+            mon: 'Mo',
+            tue: 'Di',
+            wed: 'Mi',
+            thu: 'Do',
+            fri: 'Fr',
+            sat: 'Sa'
         })
       end
 
@@ -128,12 +128,12 @@ describe Calendar do
 
     context 'when some data is missing' do
       it 'returns nil if some names format is missing' do
-        stub(TwitterCldr).get_locale_resource { { :de => { :calendars => { :gregorian => { :days => { :'stand-alone' => {} } } } } } }
+        stub(TwitterCldr).get_locale_resource { { de: { calendars: { gregorian: { days: { :'stand-alone' => {} } } } } } }
         expect(calendar.weekdays(:wide)).to eq(nil)
       end
 
       it 'returns nil if calendars data is missing' do
-        stub(TwitterCldr).get_locale_resource { { :de => {} } }
+        stub(TwitterCldr).get_locale_resource { { de: {} } }
         expect(calendar.weekdays(:wide)).to eq(nil)
       end
     end
@@ -170,8 +170,8 @@ describe Calendar do
       })
 
       expect(calendar.quarters(:narrow)).to eq({
-        1 => 1, 2 => 2,
-        3 => 3, 4 => 4
+        1 => "1", 2 => "2",
+        3 => "3", 4 => "4"
       })
     end
   end
@@ -232,12 +232,12 @@ describe Calendar do
 
   describe '#methods_for_tokens' do
     it 'converts pattern tokens into their corresponding method names' do
-      tokens = [TwitterCldr::Tokenizers::Token.new(:value => "YYYY", :type => :pattern)]
+      tokens = [TwitterCldr::Tokenizers::Token.new(value: "YYYY", type: :pattern)]
       expect(calendar.send(:methods_for_tokens, tokens)).to eq([:year_of_week_of_year])
     end
 
     it 'ignores plaintext tokens' do
-      tokens = [TwitterCldr::Tokenizers::Token.new(:value => "blarg", :type => :plaintext)]
+      tokens = [TwitterCldr::Tokenizers::Token.new(value: "blarg", type: :plaintext)]
       expect(calendar.send(:methods_for_tokens, tokens)).to eq([])
     end
   end

data/spec/shared/caser_spec.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Shared
+
+describe Caser do
+  describe '.upcase' do
+    it 'uppercases a Latin sample' do
+      str = "I like cats. They're the best."
+      expect(Caser.upcase(str)).to eq(
+        "I LIKE CATS. THEY'RE THE BEST."
+      )
+    end
+
+    it 'uppercases a Cyrillic sample' do
+      str = 'Влади́мир Влади́мирович Пу́тин'
+      expect(Caser.upcase(str)).to eq(
+        'ВЛАДИ́МИР ВЛАДИ́МИРОВИЧ ПУ́ТИН'
+      )
+    end
+
+    it 'uppercases a Greek sample' do
+      str = 'Αλφαβητικός Κατάλογος'
+      expect(Caser.upcase(str)).to eq(
+        'ΑΛΦΑΒΗΤΙΚΌΣ ΚΑΤΆΛΟΓΟΣ'
+      )
+    end
+  end
+
+  describe '.downcase' do
+    it 'lowercases a Latin sample' do
+      str = "I LIKE CATS. THEY'RE THE BEST."
+      expect(Caser.downcase(str)).to eq(
+        "i like cats. they're the best."
+      )
+    end
+
+    it 'lowercases a Cyrillic sample' do
+      str = 'ВЛАДИ́МИР ВЛАДИ́МИРОВИЧ ПУ́ТИН'
+      expect(Caser.downcase(str)).to eq(
+        'влади́мир влади́мирович пу́тин'
+      )
+    end
+
+    it 'lowercases a Greek sample' do
+      str = 'ΑΛΦΑΒΗΤΙΚΌΣ ΚΑΤΆΛΟΓΟΣ'
+      expect(Caser.downcase(str)).to eq(
+        'αλφαβητικόσ κατάλογοσ'
+      )
+    end
+  end
+
+  describe '.titlecase' do
+    it 'titlecases a Latin sample' do
+      str = 'I LIKE CATS'
+      expect(Caser.titlecase(str)).to eq(
+        'I Like Cats'
+      )
+    end
+
+    it 'lowercases a Cyrillic sample' do
+      str = 'ВЛАДИ́МИР ВЛАДИ́МИРОВИЧ ПУ́ТИН'
+      expect(Caser.titlecase(str)).to eq(
+        'Влади́мир Влади́мирович Пу́тин'
+      )
+    end
+
+    it 'lowercases a Greek sample' do
+      str = 'ΑΛΦΑΒΗΤΙΚΌΣ ΚΑΤΆΛΟΓΟΣ'
+      expect(Caser.titlecase(str)).to eq(
+        'Αλφαβητικόσ Κατάλογοσ'
+      )
+    end
+  end
+end