twitter_cldr 3.0.0.beta1 → 3.0.0

data/spec/tokenizers/calendars/date_tokenizer_spec.rb

@@ -10,35 +10,31 @@ include TwitterCldr::Tokenizers
 describe DateTokenizer do
   describe "#tokens" do
     it "should tokenize plaintext segments correctly (i.e. Spanish)" do
-      tokenizer = DateTokenizer.new(:locale => :es)
-      got       = tokenizer.tokens(:type => :full)
+      data_reader = DateDataReader.new(:es, :type => :full)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern)
       expected  = [
-          { :value => "EEEE", :type => :pattern },
-          { :value => ", ", :type => :plaintext },
-          { :value => "d", :type => :pattern },
-          { :value => " ", :type => :plaintext },
-          { :value => "'de'", :type => :plaintext },
-          { :value => " ", :type => :plaintext },
-          { :value => "MMMM", :type => :pattern },
-          { :value => " ", :type => :plaintext },
-          { :value => "'de'", :type => :plaintext },
-          { :value => " ", :type => :plaintext },
-          { :value => "y", :type => :pattern }
+        { :value => "EEEE", :type => :pattern },
+        { :value => ", ", :type => :plaintext },
+        { :value => "d", :type => :pattern },
+        { :value => " 'de' ", :type => :plaintext },
+        { :value => "MMMM", :type => :pattern },
+        { :value => " 'de' ", :type => :plaintext },
+        { :value => "y", :type => :pattern }
       ]
       check_token_list(got, expected)
     end
 
     it "should tokenize patterns with non-latin characters correctly (i.e. Japanese)" do
-      tokenizer = DateTokenizer.new(:locale => :ja)
-      got       = tokenizer.tokens(:type => :full)
+      data_reader = DateDataReader.new(:ja, :type => :full)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern)
       expected  = [
-          { :value => "y", :type => :pattern },
-          { :value => "年", :type => :plaintext },
-          { :value => "M", :type => :pattern },
-          { :value => "月", :type => :plaintext },
-          { :value => "d", :type => :pattern },
-          { :value => "日", :type => :plaintext },
-          { :value => "EEEE", :type => :pattern }
+        { :value => "y", :type => :pattern },
+        { :value => "年", :type => :plaintext },
+        { :value => "M", :type => :pattern },
+        { :value => "月", :type => :plaintext },
+        { :value => "d", :type => :pattern },
+        { :value => "日", :type => :plaintext },
+        { :value => "EEEE", :type => :pattern }
       ]
       check_token_list(got, expected)
     end
@@ -46,17 +42,17 @@ describe DateTokenizer do
     it "should tokenize composites correctly" do
       # Ensure that buddhist calendar data is present in th locale.
       TwitterCldr.get_locale_resource(:th, :calendars)[:th][:calendars][:buddhist].should_not(
-          be_nil, 'buddhist calendar is missing for :th locale (check resources/locales/th/calendars.yml)'
+        be_nil, 'buddhist calendar is missing for :th locale (check resources/locales/th/calendars.yml)'
       )
 
-      tokenizer = DateTokenizer.new(:locale => :th, :calendar_type => :buddhist)
-      got       = tokenizer.tokens(:type => :long)
+      data_reader = DateDataReader.new(:th, :type => :long, :calendar_type => :buddhist)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern)
       expected  = [
-          { :value => "d", :type => :pattern },
-          { :value => " ", :type => :plaintext },
-          { :value => "MMMM", :type => :pattern },
-          { :value => " พ.ศ. ", :type => :plaintext },
-          { :to_s => "y + 543", :type => :composite }
+        { :value => "d", :type => :pattern },
+        { :value => " ", :type => :plaintext },
+        { :value => "MMMM", :type => :pattern },
+        { :value => " พ.ศ. ", :type => :plaintext },
+        { :to_s => "y + 543", :type => :composite }
       ]
       check_token_list(got, expected)
     end

data/spec/tokenizers/calendars/datetime_tokenizer_spec.rb

@@ -8,111 +8,32 @@ require 'spec_helper'
 include TwitterCldr::Tokenizers
 
 describe DateTimeTokenizer do
-  let(:tokenizer) { DateTimeTokenizer.new }
-
-  describe "#initialize" do
-    it "chooses gregorian as the calendar type if none is specified" do
-      DateTimeTokenizer.new.calendar_type.should == :gregorian
-      DateTimeTokenizer.new(:calendar_type => :julian).calendar_type.should == :julian
-    end
-
-    it "initializes individual date and time placeholder tokenizers" do
-      placeholders = tokenizer.placeholders
-      placeholders[0][:name].should == :time
-      placeholders[0][:object].should be_a(TimeTokenizer)
-      placeholders[1][:name].should == :date
-      placeholders[1][:object].should be_a(DateTokenizer)
-    end
-  end
-
-  describe "#tokens" do
-    it "as of CLDR 23, should choose the medium date time path if no other type is specified" do
-      mock.proxy(tokenizer.paths)[:medium]
-      tokenizer.tokens
-    end
+  let(:data_reader) { DateTimeDataReader.new(:es, :type => :full) }
+  let(:tokenizer) { data_reader.tokenizer }
 
+  describe "#tokenize" do
     it "should expand date and time placeholders and return the correct list of tokens" do
-      tokenizer = DateTimeTokenizer.new(:locale => :es)
-      got       = tokenizer.tokens(:type => :full)
+      got = tokenizer.tokenize(data_reader.pattern)
       expected  = [
         { :value => "EEEE", :type => :pattern },
         { :value => ", ", :type => :plaintext },
         { :value => "d", :type => :pattern },
-        { :value => " ", :type => :plaintext },
-        { :value => "'de'", :type => :plaintext },
-        { :value => " ", :type => :plaintext },
+        { :value => " 'de' ", :type => :plaintext },
         { :value => "MMMM", :type => :pattern },
-        { :value => " ", :type => :plaintext },
-        { :value => "'de'", :type => :plaintext },
-        { :value => " ", :type => :plaintext },
+        { :value => " 'de' ", :type => :plaintext },
         { :value => "y", :type => :pattern },
+        { :value => ",", :type => :plaintext },
         { :value => " ", :type => :plaintext },
-        { :value => "HH", :type => :pattern },
+        { :value => "H", :type => :pattern },
         { :value => ":", :type => :plaintext },
         { :value => "mm", :type => :pattern },
         { :value => ":", :type => :plaintext },
         { :value => "ss", :type => :pattern },
-        { :value => " ", :type => :plaintext },
-        { :value => "zzzz", :type => :pattern }
+        { :value => " (", :type => :plaintext },
+        { :value => "zzzz", :type => :pattern },
+        { :value => ")", :type => :plaintext }
       ]
       check_token_list(got, expected)
     end
   end
-
-  describe "#mirror_resource" do
-    it "should add only the missing keys" do
-      from = {
-        :a => 1,
-        :b => { :c => 2, :d => 3 },
-        :e => { :f => 4 }
-      }
-
-      to = {
-        :b => { :c => 100 },
-        :e => 101
-      }
-
-      tokenizer.send(:mirror_resource, :from => from, :to => to)
-
-      to[:a].should == 1
-      to[:b].size.should == 2
-      to[:b][:c].should == 100
-      to[:b][:d].should == 3
-      to[:e].should == 101
-
-      from[:a].should == 1
-      from[:b].size.should == 2
-      from[:b][:c].should == 2
-      from[:b][:d].should == 3
-      from[:e].size.should == 1
-      from[:e][:f].should == 4
-    end
-  end
-
-  describe "#merge_token_type_regexes" do
-    it "merges the token type regex hash recursively, uniting regexes" do
-      first = {
-        :type1 => { :regex => /a/, :content => /aa/, :priority => 1 },
-        :type2 => { :regex => /b/, :priority => 2 },
-      }
-
-      second = {
-        :type1 => { :regex => /c/, :content => /cc/, :priority => 1 },
-        :type2 => { :regex => /d/, :priority => 2 },
-      }
-
-      tokenizer.send(:merge_token_type_regexes, first, second).should == {
-        :type1 => { :regex => Regexp.union(/a/, /c/), :content => Regexp.union(/aa/, /cc/), :priority => 1 },
-        :type2 => { :regex => Regexp.union(/b/, /d/), :priority => 2 }
-      }
-    end
-  end
-
-  describe "#pattern_for" do
-    it "returns the closest matching pattern if this tokenizer has been set up to handle additional date formats" do
-      tokenizer.instance_variable_set(:'@type', :additional)
-      tokenizer.instance_variable_set(:'@format', "MMMd")
-      tokenizer.send(:pattern_for, { :MMMd => "found me!" }).should == "found me!"
-    end
-  end
 end

data/spec/tokenizers/calendars/time_tokenizer_spec.rb

@@ -7,11 +7,11 @@ require 'spec_helper'
 
 include TwitterCldr::Tokenizers
 
-describe DateTokenizer do
+describe TimeTokenizer do
   describe "#tokens" do
     it "should tokenize a time string correctly (i.e. German)" do
-      tokenizer = TimeTokenizer.new(:locale => :de)
-      got       = tokenizer.tokens(:type => :full)
+      data_reader = TimeDataReader.new(:de, :type => :full)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern)
       expected  = [
         { :value => "HH", :type => :pattern },
         { :value => ":", :type => :plaintext },
@@ -25,8 +25,8 @@ describe DateTokenizer do
     end
 
     it "should tokenize patterns with non-latin characters correctly (i.e. Korean)" do
-      tokenizer = TimeTokenizer.new(:locale => :ko)
-      got       = tokenizer.tokens(:type => :full)
+      data_reader = TimeDataReader.new(:ko, :type => :full)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern)
       expected  = [
         { :value => "a", :type => :pattern },
         { :value => " ", :type => :plaintext },

data/spec/tokenizers/calendars/timespan_tokenizer_spec.rb

@@ -10,14 +10,24 @@ include TwitterCldr::Tokenizers
 describe TimespanTokenizer do
 
   describe "#tokens" do
-
     it "should return the correct list of tokens" do
-      tokenizer = TimespanTokenizer.new(:locale => :es)
-      got       = tokenizer.tokens(:unit => :minute, :direction => :ago, :number => 7659, :type => :default)
-
-      got[0].should == { :value => "Hace ", :type => :plaintext }
-      got[1].should == { :value => "{0}", :type => :placeholder }
-      got[2].should == { :value => " minutos", :type => :plaintext }
+      tokenizer = TimespanTokenizer.new(nil)
+      tokenizer.tokenize("Hace {0} minutos").tap do |tokens|
+        tokens[0].tap do |token|
+          token.value.should == "Hace "
+          token.type.should == :plaintext
+        end
+
+        tokens[1].tap do |token|
+          token.value.should == "{0}"
+          token.type.should == :pattern
+        end
+
+        tokens[2].tap do |token|
+          token.value.should == " minutos"
+          token.type.should == :plaintext
+        end
+      end
     end
   end
 

data/spec/tokenizers/numbers/number_tokenizer_spec.rb

@@ -8,46 +8,47 @@ require 'spec_helper'
 include TwitterCldr::Tokenizers
 
 describe NumberTokenizer do
-  describe "#full_path" do
-    it "should fill in the type and return the full path to the requested format" do
-      tokenizer = NumberTokenizer.new
-      tokenizer.type = :default
-      tokenizer.format = :default
-      tokenizer.send(:full_path).should == [:numbers, :formats, :decimal, :patterns, :default]
-    end
-  end
+  describe "#tokenize" do
+    let(:number) { 10 }
 
-  describe "#tokens" do
     it "gets tokens for a latin language (i.e. Portuguese)" do
-      tokenizer = NumberTokenizer.new(:locale => :pt)
-      got = tokenizer.tokens
-      expected = [{ :value => "", :type => :pattern },
-                  { :value => "#,##0.###", :type => :pattern }]
+      data_reader = NumberDataReader.new(:pt)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern(number))
+      expected = [
+        { :value => "", :type => :plaintext },
+        { :value => "#,##0.###", :type => :pattern }
+      ]
       check_token_list(got, expected)
     end
 
     it "gets tokens for a non-latin language (i.e. Russian)" do
-      tokenizer = NumberTokenizer.new(:locale => :ru)
-      got = tokenizer.tokens
-      expected = [{ :value => "", :type => :pattern },
-                  { :value => "#,##0.###", :type => :pattern }]
+      data_reader = NumberDataReader.new(:ru)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern(number))
+      expected = [
+        { :value => "", :type => :plaintext },
+        { :value => "#,##0.###", :type => :pattern }
+      ]
       check_token_list(got, expected)
     end
 
     it "correctly parses suffixes (i.e. Russian currency)" do
-      tokenizer = NumberTokenizer.new(:locale => :ru)
-      got = tokenizer.tokens(:type => :currency)
-      expected = [{ :value => "", :type => :pattern },
-                  { :value => "#,##0.00", :type => :pattern },
-                  { :value => " ¤", :type => :pattern }]
+      data_reader = NumberDataReader.new(:ru, :type => :currency)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern(number))
+      expected = [
+        { :value => "", :type => :plaintext },
+        { :value => "#,##0.00", :type => :pattern },
+        { :value => " ¤", :type => :plaintext }
+      ]
       check_token_list(got, expected)
     end
 
-    it "correctly parses prefixes (i.e. Italian currency)" do
-      tokenizer = NumberTokenizer.new(:locale => :it)
-      got = tokenizer.tokens(:type => :currency)
-      expected = [{ :value => "¤ ", :type => :pattern },
-                  { :value => "#,##0.00", :type => :pattern }]
+    it "correctly parses prefixes (i.e. English (American) currency)" do
+      data_reader = NumberDataReader.new(:en, :type => :currency)
+      got = data_reader.tokenizer.tokenize(data_reader.pattern(number))
+      expected = [
+        { :value => "¤", :type => :plaintext },
+        { :value => "#,##0.00", :type => :pattern }
+      ]
       check_token_list(got, expected)
     end
   end

data/spec/tokenizers/segmentation/segmentation_tokenizer_spec.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Tokenizers
+
+describe SegmentationTokenizer do
+  let(:tokenizer) { SegmentationTokenizer.new }
+
+  def tokenize(str)
+    tokenizer.tokenize(str)
+  end
+
+  describe "#tokenize" do
+    it "should tokenize an expression with a non-break" do
+      got = tokenize("$CB ÷ $SP")
+      expected = [
+        { :value => "$CB", :type => :variable },
+        { :value => "÷", :type => :break },
+        { :value => "$SP", :type => :variable }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize an expression with a non-break" do
+      got = tokenize("$ATerm × $Numeric")
+      expected = [
+        { :value => "$ATerm", :type => :variable },
+        { :value => "×", :type => :no_break },
+        { :value => "$Numeric", :type => :variable }
+      ]
+
+      check_token_list(got, expected)
+    end
+  end
+end

data/spec/tokenizers/unicode_regex/unicode_regex_tokenizer_spec.rb

@@ -0,0 +1,190 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Tokenizers
+
+describe UnicodeRegexTokenizer do
+  describe "#tokenize" do
+    let(:tokenizer) { UnicodeRegexTokenizer.new }
+
+    def tokenize(str)
+      tokenizer.tokenize(str)
+    end
+
+    it "should tokenize a regular regex" do
+      got = tokenize("^(ab)xy$")
+      expected = [
+        { :value => "^", :type => :negate },
+        { :value => "(", :type => :special_char },
+        { :value => "a", :type => :string },
+        { :value => "b", :type => :string },
+        { :value => ")", :type => :special_char },
+        { :value => "x", :type => :string },
+        { :value => "y", :type => :string },
+        { :value => "$", :type => :special_char }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing a basic character class" do
+      got = tokenize("a[bc]d")
+      expected = [
+        { :value => "a", :type => :string },
+        { :value => "[", :type => :open_bracket },
+        { :value => "b", :type => :string },
+        { :value => "c", :type => :string },
+        { :value => "]", :type => :close_bracket },
+        { :value => "d", :type => :string }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing unicode character sets" do
+      got = tokenize("\\p{Zs}[:Lu:]")
+      expected = [
+        { :value => "\\p{Zs}", :type => :character_set },
+        { :value => "[:Lu:]",  :type => :character_set }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing escaped characters" do
+      got = tokenize("^[a\\b]\\$")
+      expected = [
+        { :value => "^", :type => :negate },
+        { :value => "[", :type => :open_bracket },
+        { :value => "a", :type => :string },
+        { :value => "\\b", :type => :escaped_character },
+        { :value => "]", :type => :close_bracket },
+        { :value => "\\$", :type => :escaped_character }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing basic character ranges" do
+      got = tokenize("[a-z0-9]|[ab]")
+      expected = [
+        { :value => "[", :type => :open_bracket },
+        { :value => "a", :type => :string },
+        { :value => "-", :type => :dash },
+        { :value => "z", :type => :string },
+        { :value => "0", :type => :string },
+        { :value => "-", :type => :dash },
+        { :value => "9", :type => :string },
+        { :value => "]", :type => :close_bracket },
+        { :value => "|", :type => :pipe },
+        { :value => "[", :type => :open_bracket },
+        { :value => "a", :type => :string },
+        { :value => "b", :type => :string },
+        { :value => "]", :type => :close_bracket },
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing escaped unicode characters" do
+      got = tokenize("\\u0020[\\u0123-\\u0155]")
+      expected = [
+        { :value => "\\u0020", :type => :unicode_char },
+        { :value => "[", :type => :open_bracket },
+        { :value => "\\u0123", :type => :unicode_char },
+        { :value => "-", :type => :dash },
+        { :value => "\\u0155", :type => :unicode_char },
+        { :value => "]", :type => :close_bracket },
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing variable substitutions" do
+      got = tokenize("$CR(?:ab)[$LF]")
+      expected = [
+        { :value => "$CR", :type => :variable },
+        { :value => "(", :type => :special_char },
+        { :value => "?", :type => :special_char },
+        { :value => ":", :type => :special_char },
+        { :value => "a", :type => :string },
+        { :value => "b", :type => :string },
+        { :value => ")", :type => :special_char },
+        { :value => "[", :type => :open_bracket },
+        { :value => "$LF", :type => :variable },
+        { :value => "]", :type => :close_bracket }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing multichar strings" do
+      got = tokenize("[{foo}bar]")
+      expected = [
+        { :value => "[", :type => :open_bracket },
+        { :value => "{foo}", :type => :multichar_string },
+        { :value => "b", :type => :string },
+        { :value => "a", :type => :string },
+        { :value => "r", :type => :string },
+        { :value => "]", :type => :close_bracket }
+      ]
+    end
+
+    it "should tokenize a regex containing negated character sets" do
+      got = tokenize("[[:^N:]\\P{L}]")
+      expected = [
+        { :value => "[", :type => :open_bracket },
+        { :value => "[:^N:]", :type => :negated_character_set },
+        { :value => "\\P{L}", :type => :negated_character_set },
+        { :value => "]", :type => :close_bracket }
+      ]
+
+      check_token_list(got, expected)
+    end
+
+    it "should tokenize a regex containing some of everything" do
+      got = tokenize("^[a-zb]?[^[\\p{Z}\\u0020-\\u007f]-[\\P{L}]-[[:N:]\\u0123]][:^CC:]*[{foo}]+$")
+      expected = [
+        { :value => "^", :type => :negate },
+        { :value => "[", :type => :open_bracket },
+        { :value => "a", :type => :string },
+        { :value => "-", :type => :dash },
+        { :value => "z", :type => :string },
+        { :value => "b", :type => :string },
+        { :value => "]", :type => :close_bracket },
+        { :value => "?", :type => :special_char },
+        { :value => "[", :type => :open_bracket },
+        { :value => "^", :type => :negate },
+        { :value => "[", :type => :open_bracket },
+        { :value => "\\p{Z}", :type => :character_set },
+        { :value => "\\u0020", :type => :unicode_char },
+        { :value => "-", :type => :dash },
+        { :value => "\\u007f", :type => :unicode_char },
+        { :value => "]", :type => :close_bracket },
+        { :value => "-", :type => :dash },
+        { :value => "[", :type => :open_bracket },
+        { :value => "\\P{L}", :type => :negated_character_set },
+        { :value => "]", :type => :close_bracket },
+        { :value => "-", :type => :dash },
+        { :value => "[", :type => :open_bracket },
+        { :value => "[:N:]", :type => :character_set },
+        { :value => "\\u0123", :type => :unicode_char },
+        { :value => "]", :type => :close_bracket },
+        { :value => "]", :type => :close_bracket },
+        { :value => "[:^CC:]", :type => :negated_character_set },
+        { :value => "*", :type => :special_char },
+        { :value => "[", :type => :open_bracket },
+        { :value => "{foo}", :type => :multichar_string },
+        { :value => "]", :type => :close_bracket },
+        { :value => "+", :type => :special_char },
+        { :value => "$", :type => :special_char }
+      ]
+
+      check_token_list(got, expected)
+    end
+  end
+end