RubyGems - dhaka - Versions diffs - 2.1.0 → 2.2.0 - Mend

dhaka 2.1.0 → 2.2.0

Files changed (39) hide show

data/lib/evaluator/evaluator.rb +18 -17
data/lib/grammar/grammar.rb +4 -5
data/lib/lexer/dfa.rb +63 -13
data/lib/lexer/lexeme.rb +3 -4
data/lib/lexer/lexer.rb +12 -3
data/lib/lexer/lexer_run.rb +22 -10
data/lib/lexer/regex_grammar.rb +88 -14
data/lib/lexer/regex_parser.rb +1523 -1401
data/lib/lexer/specification.rb +29 -3
data/lib/lexer/state.rb +32 -9
data/lib/lexer/state_machine.rb +2 -2
data/lib/parser/channel.rb +4 -4
data/lib/parser/parser.rb +17 -12
data/lib/parser/parser_state.rb +3 -1
data/test/chittagong/chittagong_lexer.rb +63 -63
data/test/chittagong/chittagong_lexer.rb.rej +189 -0
data/test/chittagong/chittagong_lexer_specification.rb +6 -8
data/test/chittagong/chittagong_parser.rb +659 -659
data/test/chittagong/chittagong_parser.rb.rej +1623 -0
data/test/{another_lalr_but_not_slr_grammar.rb → core/another_lalr_but_not_slr_grammar.rb} +1 -1
data/test/{compiled_parser_test.rb → core/compiled_parser_test.rb} +1 -1
data/test/core/dfa_test.rb +170 -0
data/test/{evaluator_test.rb → core/evaluator_test.rb} +3 -3
data/test/{grammar_test.rb → core/grammar_test.rb} +3 -3
data/test/{lalr_but_not_slr_grammar.rb → core/lalr_but_not_slr_grammar.rb} +0 -0
data/test/core/lexer_test.rb +139 -0
data/test/{malformed_grammar.rb → core/malformed_grammar.rb} +0 -0
data/test/{malformed_grammar_test.rb → core/malformed_grammar_test.rb} +1 -1
data/test/{nullable_grammar.rb → core/nullable_grammar.rb} +0 -0
data/test/{parse_result_test.rb → core/parse_result_test.rb} +1 -1
data/test/{parser_state_test.rb → core/parser_state_test.rb} +1 -1
data/test/{parser_test.rb → core/parser_test.rb} +2 -2
data/test/{precedence_grammar.rb → core/precedence_grammar.rb} +0 -0
data/test/{precedence_grammar_test.rb → core/precedence_grammar_test.rb} +1 -1
data/test/{rr_conflict_grammar.rb → core/rr_conflict_grammar.rb} +0 -0
data/test/{simple_grammar.rb → core/simple_grammar.rb} +0 -0
data/test/{sr_conflict_grammar.rb → core/sr_conflict_grammar.rb} +0 -0
metadata +25 -22
data/test/lexer_test.rb +0 -215

data/test/lexer_test.rb DELETED Viewed

@@ -1,215 +0,0 @@
-require File.dirname(__FILE__) + '/dhaka_test_helper'
-class TestLexer < Test::Unit::TestCase
-  def test_build_AST_from_parse_tree_and_compute_follow_first_and_last
-    root      = Dhaka::LexerSupport::RegexParser.parse(Dhaka::LexerSupport::RegexTokenizer.tokenize("(a|b)*abb"))
-    star_node = root.left.left.left.left
-    or_node   = star_node.child
-    first_a   = or_node.children[0]
-    first_b   = or_node.children[1]
-    second_a  = root.left.left.left.right
-    second_b  = root.left.left.right
-    last_b    = root.left.right
-    sentinel  = root.right
-    assert(!root.nullable)
-    assert(!root.left.nullable)
-    assert(!root.left.left.nullable)
-    assert(star_node.nullable)
-    assert_equal(Set.new([first_a, first_b, second_a]), root.first)
-    assert_equal(Set.new([last_b]), root.left.last)
-    root.calculate_follow_sets
-    assert_equal(Set.new([first_a, first_b, second_a]), first_a.follow_set)
-    assert_equal(Set.new([first_a, first_b, second_a]), first_b.follow_set)
-    assert_equal(Set.new([second_b]), second_a.follow_set)
-    assert_equal(Set.new([last_b]), second_b.follow_set)
-    assert_equal(Set.new([sentinel]), last_b.follow_set)
-  end
-  def test_DFA_raises_exception_if_empty_regex
-    machine = Dhaka::LexerSupport::DFA.new("")
-    flunk "Should have thrown an unexpected end of regex exception"
-  rescue Dhaka::LexerSupport::InvalidRegexException => e
-    assert_equal("Unexpected end of regex.", e.message)
-  end
-  def test_DFA_raises_exception_if_error_parsing_regex
-    machine = Dhaka::LexerSupport::DFA.new("(a|b)*+abb")
-    flunk "Should have thrown an unexpected token exception"
-  rescue Dhaka::LexerSupport::InvalidRegexException => e
-    assert_equal("Unexpected token +: (a|b)*>>>+abb", e.message)
-  end
-  def test_match_a_regex
-    machine = Dhaka::LexerSupport::DFA.new("(a|b)*abb")
-    assert(machine.matches("abababb"))
-    assert(machine.matches("ababaabb"))
-    assert(!machine.matches("abababab"))
-    assert(!machine.matches("abababbc"))
-    assert(!machine.matches("abababbaa"))
-  end
-  def test_match_a_regex_with_optional_characters_at_the_end
-    machine = Dhaka::LexerSupport::DFA.new("bad(c|d)+(ab)*")
-    assert(machine.matches("badccddabab"))
-    assert(machine.matches("baddcc"))
-    assert(!machine.matches("badab"))
-    assert(!machine.matches("bacdab"))
-  end
-  def test_match_a_nullable_regex
-    machine = Dhaka::LexerSupport::DFA.new("(ab)*")
-    assert(machine.matches("abab"))
-    assert(machine.matches("ab"))
-    assert(machine.matches(""))
-    assert(!machine.matches("b"))
-  end
-  def test_match_a_regex_with_the_dot_character
-    machine = Dhaka::LexerSupport::DFA.new("ab.*cd")
-    assert(machine.matches("abacd"))
-    assert(machine.matches("abcd"))
-    assert(machine.matches("abAcd"))
-    assert(!machine.matches("ab999c"))
-  end
-  def test_match_a_regex_with_sets
-    machine = Dhaka::LexerSupport::DFA.new("ab[j-lu]*cd")
-    assert(!machine.matches("abacd"))
-    assert(machine.matches("abcd"))
-    assert(machine.matches("abjklucd"))
-    assert(!machine.matches("abijklucd"))
-    assert(!machine.matches("ab999c"))
-  end
-  def test_match_a_regex_with_negative_sets
-    machine = Dhaka::LexerSupport::DFA.new("ab[^j-lr]*cd")
-    assert(machine.matches("abcd"))
-    assert(!machine.matches("abjcd"))
-    assert(!machine.matches("abrcd"))
-    assert(!machine.matches("abijklucd"))
-    assert(machine.matches("abyqcd"))
-  end
-  def test_match_a_regex_with_sets_containing_escaped_characters
-    machine = Dhaka::LexerSupport::DFA.new("ab[\\^\\-.]*cd")
-    assert(machine.matches("abcd"))
-    assert(!machine.matches("abjcd"))
-    assert(machine.matches("ab^-.cd"))
-    assert(!machine.matches("abijklucd"))
-    assert(!machine.matches("ab\\cd"))
-  end
-  def test_match_a_regex_using_unescaped_caret_and_dash_characters
-    machine = Dhaka::LexerSupport::DFA.new("(\\^-)+")
-    assert(machine.matches("^-"))
-    assert(machine.matches("^-^-"))
-    assert(!machine.matches("?cd"))
-  end
-  def test_match_a_regex_using_escape_characters
-    machine = Dhaka::LexerSupport::DFA.new(%q/(-\?\(\)\\\\)*/)
-    assert(machine.matches("-?()\\"))
-  end
-  def test_match_a_regex_using_lt_and_gt
-    machine = Dhaka::LexerSupport::DFA.new('<.+>')
-    assert(machine.matches("<ab>"))
-    assert(machine.matches("<absdf><sdg><sse>"))
-    assert(!machine.matches("ab>"))
-  end
-  def test_simulating_curly_brace_quantifiers
-    machine = Dhaka::LexerSupport::DFA.new('aaa?a?a?')
-    assert(machine.matches("aa"))
-    assert(machine.matches("aaa"))
-    assert(machine.matches("aaaa"))
-    assert(machine.matches("aaaaa"))
-    assert(!machine.matches("aaaaaa"))
-    assert(!machine.matches("a"))
-  end
-  class LexerSpec < Dhaka::LexerSpecification
-    for_pattern 'zz' do
-      "recognized two zs"
-    end
-    for_pattern '\w(\w|\d)*' do
-      "recognized word token #{current_lexeme.value}"
-    end
-    for_pattern '(\d)*(\.\d+)?' do
-      "recognized number #{current_lexeme.value}"
-    end
-    for_pattern '<.*>' do
-      "recognized tag #{current_lexeme.value}"
-    end
-    for_pattern ' +' do
-      #ignores whitespace
-    end
-    for_pattern "\n+" do
-      "recognized newline"
-    end
-  end
-  def test_lexer_with_valid_input
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    eval(lexer.compile_to_ruby_source_as(:SomeLexer))
-    input = "these are words a z zz caPITALIZED word
-    this is a float 12.00 an integer 134 a float without a leading digit .2335 another word1"
-    results =  SomeLexer.lex(input).collect
-    assert_equal(
-    ["recognized word token these",
-     "recognized word token are",
-     "recognized word token words",
-     "recognized word token a",
-     "recognized word token z",
-     "recognized two zs",
-     "recognized word token caPITALIZED",
-     "recognized word token word",
-     "recognized newline",
-     "recognized word token this",
-     "recognized word token is",
-     "recognized word token a",
-     "recognized word token float",
-     "recognized number 12.00",
-     "recognized word token an",
-     "recognized word token integer",
-     "recognized number 134",
-     "recognized word token a",
-     "recognized word token float",
-     "recognized word token without",
-     "recognized word token a",
-     "recognized word token leading",
-     "recognized word token digit",
-     "recognized number .2335",
-     "recognized word token another",
-     "recognized word token word1"], results[0..-2])
-  end
-  def test_lexer_with_invalid_input
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    result = lexer.lex("this will cause an error here 123.").each do |result|
-    end
-    assert(result.has_error?)
-    assert_equal(34, result.unexpected_char_index)
-  end
-  def test_lexer_with_greedy_character_consumption
-    lexer = Dhaka::Lexer.new(LexerSpec)
-    results = lexer.lex("<html></html>this is a word").collect
-    assert_equal(["recognized tag <html></html>",
-     "recognized word token this",
-     "recognized word token is",
-     "recognized word token a",
-     "recognized word token word"], results[0..-2])
-  end
-end