twitter_cldr 3.0.0.beta1 → 3.0.0

data/spec/parsers/number_parser_spec.rb

@@ -7,7 +7,7 @@ require 'spec_helper'
 
 include TwitterCldr::Parsers
 
-describe TwitterCldr::Parsers::NumberParser do
+describe NumberParser do
   let(:separators) { ["\\.", ","] }
 
   before(:each) do
@@ -16,7 +16,7 @@ describe TwitterCldr::Parsers::NumberParser do
 
   describe "#group_separator" do
     it "returns the correct group separator" do
-      @parser.send(:group_separator).should == "\\."
+      @parser.send(:group_separator).should match_normalized(" ")
     end
   end
 
@@ -69,7 +69,7 @@ describe TwitterCldr::Parsers::NumberParser do
 
     it "returns only locale-specific separators when strict mode is on" do
       group, decimal = @parser.send(:separators, true)
-      group.should == '\.'
+      group.should match_normalized(" ")
       decimal.should == ','
     end
   end
@@ -120,7 +120,7 @@ describe TwitterCldr::Parsers::NumberParser do
     end
 
     it "correctly identifies a series of invalid cases" do
-      ["12,0,0", "5,", "5."].each do |num|
+      ["12,0,0", "5,", "5#{[160].pack("U*")}"].each do |num|
         @parser.valid?(num).should be_false
       end
     end
@@ -144,7 +144,7 @@ describe TwitterCldr::Parsers::NumberParser do
     end
 
     it "correctly raises an error when asked to parse invalid numbers" do
-      cases = ["12,0,0", "5,", "5."]
+      cases = ["12,0,0", "5,", "5#{[160].pack("U*")}"]
       cases.each do |text|
         lambda { @parser.parse(text) }.should raise_error(InvalidNumberError)
       end

data/spec/parsers/parser_spec.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+class FakeParser < Parser
+  def do_parse(options); end
+end
+
+describe Parser do
+  let(:parser) { FakeParser.new }
+  let(:tokens) do
+    [
+      Token.new(:type => :a, :value => "a"),
+      Token.new(:type => :b, :value => "b"),
+      Token.new(:type => :c, :value => "c")
+    ]
+  end
+
+  describe "#parse" do
+    it "should call do_parse" do
+      mock(parser).do_parse({})
+      parser.parse(tokens)
+    end
+  end
+
+  describe "#reset" do
+    it "should reset the token index" do
+      parser.parse(tokens)
+      parser.send(:next_token, :a)
+      parser.send(:current_token).type.should == :b
+      parser.reset
+      parser.send(:current_token).type.should == :a
+    end
+  end
+
+  describe "#next_token" do
+    it "should advance to the next token" do
+      parser.parse(tokens)
+      parser.send(:next_token, :a)
+      parser.send(:current_token).type.should == :b
+    end
+
+    it "should raise an error after encountering an unexpected token" do
+      parser.parse(tokens)
+      lambda { parser.send(:next_token, :z) }.should raise_error(UnexpectedTokenError)
+    end
+  end
+
+  describe "#current_token" do
+    it "returns the current token" do
+      parser.parse(tokens)
+      parser.send(:current_token).type.should == :a
+    end
+  end
+end

data/spec/parsers/segmentation_parser_spec.rb

@@ -0,0 +1,96 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+include TwitterCldr::Tokenizers
+
+describe "Segmentation" do
+  let(:tokenizer) { SegmentationTokenizer.new }
+  let(:parser) { SegmentationParser.new }
+
+  def tokenize(str)
+    tokenizer.tokenize(str)
+  end
+
+  def parse(tokens, options = {})
+    parser.parse(tokens, options)
+  end
+
+  describe SegmentationParser do
+    let(:symbol_table) do
+      SymbolTable.new({
+        "$FOO" => tokenize("[abc]")
+      })
+    end
+
+    describe "#parse" do
+      it "should parse a rule with a break" do
+        rule = parse(tokenize("[a-z] ÷ [0-9]"))
+        rule.left.to_regexp_str.should == "\\A(?:[\\141-\\172])"
+        rule.right.to_regexp_str.should == "\\A(?:[\\60-\\71])"
+        rule.boundary_symbol.should == :break
+      end
+
+      it "should parse a rule with a non-break" do
+        rule = parse(tokenize("[a-z] × [0-9]"))
+        rule.regex.to_regexp_str.should == "\\A(?:[\\141-\\172])(?:[\\60-\\71])"
+        rule.boundary_symbol.should == :no_break
+      end
+
+      it "should parse a rule containing a variable" do
+        rule = parse(tokenize("$FOO × bar"), :symbol_table => symbol_table)
+        rule.regex.to_regexp_str.should == "\\A(?:[\\141-\\143])(?:\\142)(?:\\141)(?:\\162)"
+        rule.boundary_symbol.should == :no_break
+      end
+    end
+  end
+
+  describe SegmentationParser::BreakRule do
+    describe "#match" do
+      let(:rule) { parse(tokenize("[a-z] ÷ [0-9]")) }
+
+      it "rule should be the right type" do
+        rule.should be_a(SegmentationParser::BreakRule)
+      end
+
+      it "should match and return the right offset and text" do
+        match = rule.match("c7")
+        match.boundary_offset.should == 1
+        match.text.should == "c7"
+      end
+
+      it "should not match if the input string doesn't contain a matching right- and/or left-hand side" do
+        rule.match("C7").should be_nil
+        rule.match("cc").should be_nil
+        rule.match("CC").should be_nil
+      end
+    end
+  end
+
+  describe SegmentationParser::NoBreakRule do
+    describe "#match" do
+      let(:rule) { parse(tokenize("[a-z] × [0-9]")) }
+
+      it "rule should be the right type" do
+        rule.should be_a(SegmentationParser::NoBreakRule)
+      end
+
+      it "should match and return the right offset and text" do
+        match = rule.match("c7")
+        # non-break rules send you to the end of the match (maybe that's wrong?)
+        match.boundary_offset.should == 2
+        match.text.should == "c7"
+      end
+
+      it "should not match if the input string doesn't contain matching text" do
+        rule.match("C7").should be_nil
+        rule.match("cc").should be_nil
+        rule.match("CC").should be_nil
+      end
+    end
+  end
+end

data/spec/parsers/symbol_table_spec.rb

@@ -0,0 +1,32 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe SymbolTable do
+  let(:table) { SymbolTable.new(:a => "b", :c => "d") }
+
+  describe "#fetch" do
+    it "should be able to retrieve values for symbols" do
+      table.fetch(:a).should == "b"
+      fetch = lambda { table.fetch(:z) }
+
+      if RUBY_VERSION > "1.8.7"
+        fetch.should raise_error(KeyError)
+      else
+        fetch.should raise_error(IndexError)
+      end
+    end
+  end
+
+  describe "#add" do
+    it "should be able to add then fetch new values for symbols" do
+      table.add(:e, "f")
+      table.fetch(:e).should == "f"
+    end
+  end
+end

data/spec/parsers/unicode_regex/character_class_spec.rb

@@ -0,0 +1,117 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe UnicodeRegexParser::CharacterClass do
+  let(:tokenizer) { UnicodeRegexTokenizer.new }
+  let(:parser) { UnicodeRegexParser.new }
+
+  def tokenize(str)
+    tokenizer.tokenize(str)
+  end
+
+  def parse(tokens, options = {})
+    parser.parse(tokens, options)
+  end
+
+  def char_class_from(elements)
+    elements.first
+  end
+
+  describe "#to_set" do
+    it "unions together char classes with no explicit operator" do
+      char_class = char_class_from(parse(tokenize("[[a][b]]")))
+      char_class.to_set.to_a.should == [97..98]
+    end
+
+    it "unions together other entities within char classes when operator is not explicit" do
+      char_class = char_class_from(parse(tokenize("[a-z0-9\\u0123]")))
+      char_class.to_set.to_a(true).should == [48..57, 97..122, 291]
+    end
+
+    it "intersects correctly" do
+      char_class = char_class_from(parse(tokenize("[[a-m]&[g-z]]")))
+      char_class.to_set.to_a.should == [103..109]
+    end
+
+    it "finds symmetric differences correctly" do
+      char_class = char_class_from(parse(tokenize("[[a-m]-[g-z]]")))
+      char_class.to_set.to_a.should == [97..102, 110..122]
+    end
+
+    it "computes sets for nested expressions" do
+      # (97..109) U (104..106)
+      # = (104..106)
+      # ((104..106) U (107..122)) subtr ((104..106) C (107..122))
+      # = (104..122) subtr ()
+      # = (104..122)
+      char_class = char_class_from(parse(tokenize("[[[a-m]&[h-j]]-[k-z]]")))
+      char_class.to_set.to_a.should == [104..122]
+    end
+
+    it "pulls in ranges for unicode character sets" do
+      char_class = char_class_from(parse(tokenize("[\\p{Zs}]")))
+      char_class.to_set.to_a(true).should == [
+        32, 160, 5760, 6158, 8192..8202, 8239, 8287, 12288
+      ]
+    end
+
+    it "computes unions between unicode character sets" do
+      char_class = char_class_from(parse(tokenize("[[\\p{Zs}][\\p{Cc}]]")))
+      char_class.to_set.to_a(true).should == [
+        0..1, 8..32, 127..160, 5760, 6158, 8192..8202, 8239, 8287, 12288
+      ]
+    end
+
+    it "computes intersections between unicode character sets" do
+      char_class = char_class_from(parse(tokenize("[[\\p{Zs}]&[\\u2000-\\u202B]]")))
+      char_class.to_set.to_a(true).should == [8192..8202]
+    end
+
+    it "supports negating character sets" do
+      char_class = char_class_from(parse(tokenize("[^\\u2000-\\u202B]")))
+      char_class.to_set.to_a(true).should == [
+        0..1, 8..8191, 8236..55295, 57344..1114111
+      ]
+    end
+
+    it "supports literal and escaped characters" do
+      char_class = char_class_from(parse(tokenize("[abc\\edf\\g]")))
+      char_class.to_set.to_a(true).should == [97..103]
+    end
+
+    it "supports special switch characters" do
+      char_class = char_class_from(parse(tokenize("[\\w]")))  # a-z, A-Z, 0-9, _
+      char_class.to_set.to_a(true).should == [48..57, 65..90, 95, 97..122]
+    end
+
+    it "supports negated switch characters" do
+      char_class = char_class_from(parse(tokenize("[\\D]")))  # i.e. NOT \w
+      char_class.to_set.to_a(true).should == [
+        0..1, 8..47, 58..55295, 57344..1114111
+      ]
+    end
+  end
+
+  describe "#to_regexp_str" do
+    it "wraps ranges in square brackets" do
+      char_class = char_class_from(parse(tokenize("[a-z]")))
+      char_class.to_regexp_str.should == "(?:[\\141-\\172])"
+    end
+
+    it "octal-encodes and wraps sequential characters to isolate bytes" do
+      char_class = char_class_from(parse(tokenize("[{foo}]")))
+      char_class.to_regexp_str.should == "(?:(?:\\146)(?:\\157)(?:\\157))"
+    end
+
+    it "combines multiple components with 'or' pipe characters" do
+      char_class = char_class_from(parse(tokenize("[{foo}abc]")))
+      char_class.to_regexp_str.should == "(?:(?:\\146)(?:\\157)(?:\\157)|[\\141-\\143])"
+    end
+  end
+end

data/spec/parsers/unicode_regex/character_range_spec.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe UnicodeRegexParser::CharacterRange do
+  describe "#to_set" do
+    it "should return a range between the initial and the final values" do
+      range = UnicodeRegexParser::CharacterRange.new(
+        UnicodeRegexParser::UnicodeString.new([97]),
+        UnicodeRegexParser::UnicodeString.new([98])
+      )
+
+      range.to_set.to_a(true).should == [97..98]
+    end
+  end
+end

data/spec/parsers/unicode_regex/character_set_spec.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe UnicodeRegexParser::CharacterSet do
+  describe "#to_set" do
+    it "should return a set containing codepoints for the given general property" do
+      char_set = UnicodeRegexParser::CharacterSet.new("Zs")
+      char_set.to_set.to_a(true).should == [
+        32, 160, 5760, 6158, 8192..8202, 8239, 8287, 12288
+      ]
+    end
+
+    it "should return a set containing codepoints for the given named property" do
+      char_set = UnicodeRegexParser::CharacterSet.new("Sentence_Break=Sp")
+      char_set.to_set.to_a(true).should == [
+        9, 11..12, 32, 160, 5760, 8192..8202, 8239, 8287, 12288
+      ]
+    end
+
+    it "should raise an exception when given an invalid property name or value" do
+      lambda do
+        UnicodeRegexParser::CharacterSet.new("Foobar=Sp").to_set
+      end.should raise_error(UnicodeRegexParserError)
+
+      lambda do
+        UnicodeRegexParser::CharacterSet.new("Sentence_Break=Foo").to_set
+      end.should raise_error(UnicodeRegexParserError)
+    end
+  end
+end

data/spec/parsers/unicode_regex/literal_spec.rb

@@ -0,0 +1,34 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe UnicodeRegexParser::Literal do
+  describe "#to_set" do
+    it "should return literal characters as codepoints" do
+      literal = UnicodeRegexParser::Literal.new("a")
+      literal.to_set.to_a(true).should == [97]
+    end
+
+    it "should return escaped characters with no special meaning as codepoints" do
+      literal = UnicodeRegexParser::Literal.new("\\a")
+      literal.to_set.to_a(true).should == [97]
+    end
+
+    it "should convert special regex switches to their range equivalents" do
+      literal = UnicodeRegexParser::Literal.new("\\d")  # digit
+      literal.to_set.to_a(true).should == [48..57]
+    end
+
+    it "should convert negated special regex switches to their range equivalents" do
+      literal = UnicodeRegexParser::Literal.new("\\D")  # NOT digit
+      literal.to_set.to_a(true).should == [
+        0..1, 8..47, 58..55295, 57344..1114111
+      ]
+    end
+  end
+end

data/spec/parsers/unicode_regex/unicode_string_spec.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+
+describe UnicodeRegexParser::UnicodeString do
+  describe "#to_set" do
+    it "should return a zero-length range when representing a single codepoint" do
+      str = UnicodeRegexParser::UnicodeString.new([97])
+      str.to_set.to_a.should == [97..97]
+    end
+
+    it "should return a range containing the codepoint array as both the first and last elements" do
+      str = UnicodeRegexParser::UnicodeString.new([97, 98, 99])
+      str.to_set.to_a.should == [[97, 98, 99]..[97, 98, 99]]
+    end
+  end
+end

data/spec/parsers/unicode_regex_parser_spec.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+include TwitterCldr::Parsers
+include TwitterCldr::Tokenizers
+
+describe UnicodeRegexParser do
+  let(:tokenizer) { UnicodeRegexTokenizer.new }
+  let(:parser) { UnicodeRegexParser.new }
+
+  def tokenize(str)
+    tokenizer.tokenize(str)
+  end
+
+  def parse(tokens, options = {})
+    parser.parse(tokens, options)
+  end
+
+  describe "#parse" do
+    it "identifies ranges" do
+      elements = parse(tokenize("[a-z]"))
+      elements.first.should be_a(UnicodeRegexParser::CharacterClass)
+      root = elements.first.send(:root)
+      root.should be_a(UnicodeRegexParser::CharacterRange)
+      root.initial.codepoints.should == "a".unpack("U*")
+      root.final.codepoints.should == "z".unpack("U*")
+    end
+
+    it "replaces variables" do
+      symbol_table = SymbolTable.new("$VAR" => tokenize("\\p{L}"))
+      elements = parse(tokenize("($VAR)?"), :symbol_table => symbol_table)
+      elements[1].should be_a(UnicodeRegexParser::CharacterSet)
+      elements[1].property_value.should == "L"
+    end
+
+    it "handles character and negated character sets" do
+      elements = parse(tokenize("\\p{L}[:^P:]\\P{L}[:P:]"))
+
+      element = elements[0]
+      element.should be_a(UnicodeRegexParser::CharacterSet)
+      element.property_value.should == "L"
+
+      element = elements[1]
+      element.should be_a(UnicodeRegexParser::CharacterClass)
+      element.send(:root).child.property_value.should == "P"
+      element.send(:root).operator.should == :negate
+
+      element = elements[2]
+      element.should be_a(UnicodeRegexParser::CharacterClass)
+      element.send(:root).child.property_value.should == "L"
+
+      element = elements[3]
+      element.should be_a(UnicodeRegexParser::CharacterSet)
+      element.property_value.should == "P"
+    end
+
+    it "handles unicode characters" do
+      elements = parse(tokenize("\\u0123"))
+      elements[0].should be_a(UnicodeRegexParser::UnicodeString)
+      elements[0].codepoints.should == [291]
+    end
+
+    it "handles multichar and escaped unicode strings" do
+      elements = parse(tokenize("\\g{abc}"))
+      elements[0].should be_a(UnicodeRegexParser::Literal)
+      elements[0].text.should == "\\g"
+      elements[1].should be_a(UnicodeRegexParser::UnicodeString)
+      elements[1].codepoints.should == [97, 98, 99]
+    end
+
+    it "handles special chars" do
+      elements = parse(tokenize("^(?:)$"))
+      elements.each { |elem| elem.should be_a(UnicodeRegexParser::Literal) }
+      elements[0].text.should == "^"
+      elements[1].text.should == "("
+      elements[2].text.should == "?"
+      elements[3].text.should == ":"
+      elements[4].text.should == ")"
+      elements[5].text.should == "$"
+    end
+  end
+end