dhaka 2.1.0 → 2.2.0

data/test/lexer_test.rb

@@ -1,215 +0,0 @@
-require File.dirname(__FILE__) + '/dhaka_test_helper'
-
-class TestLexer < Test::Unit::TestCase
-  def test_build_AST_from_parse_tree_and_compute_follow_first_and_last
-    root      = Dhaka::LexerSupport::RegexParser.parse(Dhaka::LexerSupport::RegexTokenizer.tokenize("(a|b)*abb"))
-    star_node = root.left.left.left.left
-    or_node   = star_node.child
-    first_a   = or_node.children[0]
-    first_b   = or_node.children[1]
-    second_a  = root.left.left.left.right
-    second_b  = root.left.left.right
-    last_b    = root.left.right
-    sentinel  = root.right
-    
-    assert(!root.nullable)
-    assert(!root.left.nullable)
-    assert(!root.left.left.nullable)
-    assert(star_node.nullable)
-    
-    assert_equal(Set.new([first_a, first_b, second_a]), root.first)
-    assert_equal(Set.new([last_b]), root.left.last)
-    
-    root.calculate_follow_sets
-    
-    assert_equal(Set.new([first_a, first_b, second_a]), first_a.follow_set)
-    assert_equal(Set.new([first_a, first_b, second_a]), first_b.follow_set)
-    assert_equal(Set.new([second_b]), second_a.follow_set)
-    assert_equal(Set.new([last_b]), second_b.follow_set)
-    assert_equal(Set.new([sentinel]), last_b.follow_set)
-  end
-
-  def test_DFA_raises_exception_if_empty_regex
-    machine = Dhaka::LexerSupport::DFA.new("")
-    flunk "Should have thrown an unexpected end of regex exception"
-  rescue Dhaka::LexerSupport::InvalidRegexException => e
-    assert_equal("Unexpected end of regex.", e.message)
-  end
-
-  def test_DFA_raises_exception_if_error_parsing_regex
-    machine = Dhaka::LexerSupport::DFA.new("(a|b)*+abb")
-    flunk "Should have thrown an unexpected token exception"
-  rescue Dhaka::LexerSupport::InvalidRegexException => e
-    assert_equal("Unexpected token +: (a|b)*>>>+abb", e.message)
-  end
-  
-  def test_match_a_regex
-    machine = Dhaka::LexerSupport::DFA.new("(a|b)*abb")
-    assert(machine.matches("abababb"))
-    assert(machine.matches("ababaabb"))
-    assert(!machine.matches("abababab"))
-    assert(!machine.matches("abababbc"))
-    assert(!machine.matches("abababbaa"))
-  end
-
-  def test_match_a_regex_with_optional_characters_at_the_end
-    machine = Dhaka::LexerSupport::DFA.new("bad(c|d)+(ab)*")
-    assert(machine.matches("badccddabab"))
-    assert(machine.matches("baddcc"))
-    assert(!machine.matches("badab"))
-    assert(!machine.matches("bacdab"))
-  end
-
-  def test_match_a_nullable_regex
-    machine = Dhaka::LexerSupport::DFA.new("(ab)*")
-    assert(machine.matches("abab"))
-    assert(machine.matches("ab"))
-    assert(machine.matches(""))
-    assert(!machine.matches("b"))
-  end
-
-  def test_match_a_regex_with_the_dot_character
-    machine = Dhaka::LexerSupport::DFA.new("ab.*cd")
-    assert(machine.matches("abacd"))
-    assert(machine.matches("abcd"))
-    assert(machine.matches("abAcd"))
-    assert(!machine.matches("ab999c"))
-  end
-  
-  def test_match_a_regex_with_sets
-    machine = Dhaka::LexerSupport::DFA.new("ab[j-lu]*cd")
-    assert(!machine.matches("abacd"))
-    assert(machine.matches("abcd"))
-    assert(machine.matches("abjklucd"))
-    assert(!machine.matches("abijklucd"))
-    assert(!machine.matches("ab999c"))
-  end
-
-  def test_match_a_regex_with_negative_sets
-    machine = Dhaka::LexerSupport::DFA.new("ab[^j-lr]*cd")
-    assert(machine.matches("abcd"))
-    assert(!machine.matches("abjcd"))
-    assert(!machine.matches("abrcd"))
-    assert(!machine.matches("abijklucd"))
-    assert(machine.matches("abyqcd"))
-  end
-  
-  def test_match_a_regex_with_sets_containing_escaped_characters
-    machine = Dhaka::LexerSupport::DFA.new("ab[\\^\\-.]*cd")
-    assert(machine.matches("abcd"))
-    assert(!machine.matches("abjcd"))
-    assert(machine.matches("ab^-.cd"))
-    assert(!machine.matches("abijklucd"))
-    assert(!machine.matches("ab\\cd"))
-  end
-
-  def test_match_a_regex_using_unescaped_caret_and_dash_characters
-    machine = Dhaka::LexerSupport::DFA.new("(\\^-)+")
-    assert(machine.matches("^-"))
-    assert(machine.matches("^-^-"))
-    assert(!machine.matches("?cd"))
-  end
-  
-  def test_match_a_regex_using_escape_characters
-    machine = Dhaka::LexerSupport::DFA.new(%q/(-\?\(\)\\\\)*/)
-    assert(machine.matches("-?()\\"))
-  end
-
-  def test_match_a_regex_using_lt_and_gt
-    machine = Dhaka::LexerSupport::DFA.new('<.+>')
-    assert(machine.matches("<ab>"))
-    assert(machine.matches("<absdf><sdg><sse>"))
-    assert(!machine.matches("ab>"))
-  end
-  
-  def test_simulating_curly_brace_quantifiers
-    machine = Dhaka::LexerSupport::DFA.new('aaa?a?a?')
-    assert(machine.matches("aa"))
-    assert(machine.matches("aaa"))
-    assert(machine.matches("aaaa"))
-    assert(machine.matches("aaaaa"))
-    assert(!machine.matches("aaaaaa"))
-    assert(!machine.matches("a"))
-  end
-  
-  class LexerSpec < Dhaka::LexerSpecification
-
-    for_pattern 'zz' do
-      "recognized two zs"
-    end
-    
-    for_pattern '\w(\w|\d)*' do
-      "recognized word token #{current_lexeme.value}"
-    end
-
-    for_pattern '(\d)*(\.\d+)?' do
-      "recognized number #{current_lexeme.value}"
-    end
-    
-    for_pattern '<.*>' do
-      "recognized tag #{current_lexeme.value}"
-    end
-
-    for_pattern ' +' do
-      #ignores whitespace
-    end
-    
-    for_pattern "\n+" do
-      "recognized newline"
-    end
-  end
-  
-  def test_lexer_with_valid_input
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    eval(lexer.compile_to_ruby_source_as(:SomeLexer))
-    input = "these are words a z zz caPITALIZED word     
-    this is a float 12.00 an integer 134 a float without a leading digit .2335 another word1"
-    results =  SomeLexer.lex(input).collect
-    assert_equal(
-    ["recognized word token these",
-     "recognized word token are",
-     "recognized word token words",
-     "recognized word token a",
-     "recognized word token z",
-     "recognized two zs",
-     "recognized word token caPITALIZED",
-     "recognized word token word",
-     "recognized newline",
-     "recognized word token this",
-     "recognized word token is",
-     "recognized word token a",
-     "recognized word token float",
-     "recognized number 12.00",
-     "recognized word token an",
-     "recognized word token integer",
-     "recognized number 134",
-     "recognized word token a",
-     "recognized word token float",
-     "recognized word token without",
-     "recognized word token a",
-     "recognized word token leading",
-     "recognized word token digit",
-     "recognized number .2335",
-     "recognized word token another",
-     "recognized word token word1"], results[0..-2])
-  end
-  
-  def test_lexer_with_invalid_input
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    result = lexer.lex("this will cause an error here 123.").each do |result| 
-    end
-    assert(result.has_error?)
-    assert_equal(34, result.unexpected_char_index)
-  end
-
-  def test_lexer_with_greedy_character_consumption
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    results = lexer.lex("<html></html>this is a word").collect
-    assert_equal(["recognized tag <html></html>",
-     "recognized word token this",
-     "recognized word token is",
-     "recognized word token a",
-     "recognized word token word"], results[0..-2])
-  end
-
-end