aurum 0.1.1 → 0.2.0

data/spec/grammar/dsl/precedence_builder_spec.rb

@@ -0,0 +1,64 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::DSL::PrecedencesBuilder do
+  before :each do
+    @syntax_rules = Aurum::Grammar::SyntaxRules.new
+    @builder = Aurum::Grammar::DSL::PrecedencesBuilder.new(@syntax_rules)
+  end
+  
+  it 'string should be declared as literal non-associative operator' do
+    @builder.nonassoc '+'
+    precedence = @syntax_rules.operator_precedence('$literal_+')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, 0))
+  end
+  
+  it 'symbol should be declared as terminal non-associative operator' do
+    @builder.nonassoc :_id
+    precedence = @syntax_rules.operator_precedence('_id')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, 0))
+  end
+  
+  it 'string should be declared as literal left-associative operator' do
+    @builder.left '+'
+    precedence = @syntax_rules.operator_precedence('$literal_+')
+    precedence.should eql?(Aurum::Grammar.precedence(:left_associative, 0))
+  end
+  
+  it 'symbol should be declared as terminal left-associative operator' do
+    @builder.left :_id
+    precedence = @syntax_rules.operator_precedence('_id')
+    precedence.should eql?(Aurum::Grammar.precedence(:left_associative, 0))
+  end  
+  
+  it 'string should be declared as literal right-associative operator' do
+    @builder.right '+'
+    precedence = @syntax_rules.operator_precedence('$literal_+')
+    precedence.should eql?(Aurum::Grammar.precedence(:right_associative, 0))
+  end
+  
+  it 'symbol should be declared as terminal right-associative operator' do
+    @builder.right :_id
+    precedence = @syntax_rules.operator_precedence('_id')
+    precedence.should eql?(Aurum::Grammar.precedence(:right_associative, 0))
+  end    
+  
+  it 'operators declared in the same time should have same precedence' do
+    @builder.nonassoc '+', '-'
+    precedence = @syntax_rules.operator_precedence('$literal_+')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, 0))
+    precedence = @syntax_rules.operator_precedence('$literal_-')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, 0))
+  end
+  
+  it 'operators declared earlier should have higher precedence' do
+    @builder.nonassoc '+'
+    @builder.nonassoc '-'
+    precedence = @syntax_rules.operator_precedence('$literal_+')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, 0))
+    precedence = @syntax_rules.operator_precedence('$literal_-')
+    precedence.should eql?(Aurum::Grammar.precedence(:non_associative, -1))
+  end
+end

data/spec/grammar/dsl/productions_builder_spec.rb

@@ -0,0 +1,78 @@
+helper_dir = File.join(File.dirname(__FILE__), '..', '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::DSL::ProductionsBuilder do
+  before :each do
+    @syntax_rules, @actions = Aurum::Grammar::SyntaxRules.new, {}
+    @builder = Aurum::Grammar::DSL::ProductionsBuilder.new(@syntax_rules, @actions)
+  end
+
+  it 'should add terminal to terminal list' do
+    @builder.instance_eval do
+      nonterminal _terminal
+    end
+    terminals = @syntax_rules.instance_eval {@terminals}
+    terminals.size.should == 1
+    terminals['_terminal'].should == Aurum::Grammar.terminal('_terminal')
+  end
+  
+  it 'should add literal to terminal and literal list' do
+    @builder.instance_eval do
+      nonterminal '+'
+    end
+    terminals, literals = @syntax_rules.instance_eval {[@terminals, @literals]}
+    terminals.size.should == 1
+    terminals['$literal_+'].should == Aurum::Grammar.terminal('$literal_+')
+    literals.should == ['+'].to_set
+  end
+
+  it 'should add production to nonterminal' do
+    @builder.instance_eval do
+      nonterminal _terminal
+    end
+    productions = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal'))
+    productions.map {|prod| prod.inspect}.join(' , ').should == 'nonterminal -> _terminal'
+  end
+
+  it 'should add empty production to nonterminal' do
+    @builder.instance_eval do
+      nonterminal _
+    end
+    productions = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal'))
+    productions.map {|prod| prod.inspect}.join(' , ').should == 'nonterminal -> '
+  end
+  
+  it 'should use object_id as anonymouse production name' do
+    @builder.instance_eval do
+      nonterminal _terminal
+    end
+    production = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal')).to_a[0]
+    production.name.should == "$production_#{production.object_id}"
+  end
+  
+  it 'should use last symbol as production name' do
+    @builder.instance_eval do
+      nonterminal _terminal, :production_name
+    end
+    production = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal')).to_a[0]
+    production.name.should == 'production_name'
+  end
+
+  it 'should add action to actions table for anonymouse production' do
+    @builder.instance_eval do
+      nonterminal _terminal { 'action' }
+    end
+    production = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal')).to_a[0]
+    @actions[production.name].should_not be_nil
+  end
+
+  it 'should add action to actions table for named production' do
+    @builder.instance_eval do
+      nonterminal _terminal, :name do 'action' end
+    end
+    production = @syntax_rules.productions(Aurum::Grammar.nonterminal('nonterminal')).to_a[0]
+    @actions[production.name].should_not be_nil
+  end
+end

data/spec/grammar/precedence_spec.rb

@@ -0,0 +1,42 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::SyntaxRules::Precedence do
+  it 'precedence2 should greater than precedence2 if operator1 has higher precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative,  0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, -1)
+    precedence1.should > precedence2
+  end
+
+  it 'precedence2 should less than precedence1 if precedence1 has higher precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative,  0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, -1)
+    precedence2.should < precedence1
+  end
+  
+  it 'precedence1 should greater then precedence2 if operators are left-associative and same precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, 0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, 0)
+    precedence2.should > precedence1
+  end
+  
+  it 'precedence2 should greater then precedence1 if operators are left-associative and same precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, 0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:left_associative, 0)
+    precedence1.should > precedence2
+  end  
+  
+  it 'precedence1 should less then precedence2 if operators are left-associative and same precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:right_associative, 0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:right_associative, 0)
+    precedence1.should < precedence2
+  end
+  
+  it 'precedence2 should less then precedence1 if operators are left-associative and same precedence' do
+    precedence1 = Aurum::Grammar::SyntaxRules::Precedence.new(:right_associative, 0)
+    precedence2 = Aurum::Grammar::SyntaxRules::Precedence.new(:right_associative, 0)
+    precedence2.should < precedence1
+  end    
+end

data/spec/grammar/syntax_rules_spec.rb

@@ -0,0 +1,31 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Grammar::SyntaxRules do
+  before :all do
+    @syntax_rules = Aurum::Grammar::SyntaxRules.new
+    @E, @T, @plus = Aurum::Grammar.nonterminal('E'), Aurum::Grammar.nonterminal('T'), Aurum::Grammar.terminal('$literal_+')
+    @syntax_rules.add_syntax_rule Aurum::Grammar.production(@E, [@E, @plus, @E])
+    @syntax_rules.add_syntax_rule Aurum::Grammar.production(@E, [@T, @plus, @T])
+  end
+  
+  it 'should add productions to nonterminal' do    
+    productions = @syntax_rules.productions(@E).map {|production| production.inspect}.to_set
+    productions.should == ['E -> T + T', 'E -> E + E'].to_set
+  end
+  
+  it 'should not add duplicated productions' do
+    @syntax_rules.add_syntax_rule Aurum::Grammar.production(@E, [@T, @plus, @T])
+    productions = @syntax_rules.productions(@E).map {|production| production.inspect}.to_set
+    productions.size.should == 2
+  end
+  
+  it 'should assign precedence to operator symbol' do
+    precedence = Aurum::Grammar.precedence(:left_associative, 1)
+    @syntax_rules.add_operator_precedence('$literal_+', precedence)
+    @syntax_rules.assign_operator_precedence_to_symbols
+    @syntax_rules.instance_eval {@terminals['$literal_+'].precedence}.should == precedence
+  end
+end

data/spec/parser_matcher.rb

@@ -0,0 +1,69 @@
+module ParserMatcher
+  def token(terminal, value = terminal)
+    Aurum::Engine::Token.new(terminal.to_s, value)
+  end
+
+  def literal(terminal, value = terminal)
+    Aurum::Engine::Token.new("$literal_#{terminal}", value)
+  end
+
+  def be_parsed_as(nonterminal, value=nil)
+    parser_creator = Aurum::Parser.new(@grammar, nonterminal)
+    BeParsedAs.new(parser_creator.new, nonterminal, value)
+  end
+
+  class BeParsedAs
+    def initialize parser, nonterminal, value
+      @parser, @nonterminal, @value = parser, nonterminal, value
+      class << @parser
+        def error_recovery
+          raise 'error_recover'
+        end
+      end
+    end
+
+    def matches?(source)
+      @source = source
+      (result = @parser.parse(Lexer.new(source))) rescue return false
+      return true unless @value
+      @value.all? {|attr, value| result.__send__(attr) == value}
+    end
+
+    def failure_message
+      "[#{@source.join(', ')}] can't be parsed as #{@nonterminal}"
+    end
+
+    def negative_failure_message
+      "[#{@source.join(', ')}] can be parsed as #{@nonterminal}"
+    end
+
+    def description
+      "be parsed as #{@nonterminal} if [#{@source.join(', ')}] given"
+    end
+  end
+
+  class Lexer
+    def initialize source
+      @source = source.map do |token|
+        case token
+        when String : Aurum::Engine::Token.new("$literal_#{token}", token)
+        when Symbol : Aurum::Engine::Token.new(token.to_s, token)
+        when Aurum::Engine::Token : token
+        end
+      end
+    end
+
+    def next_symbol
+      symbol = @source.shift()
+      symbol ? symbol :  Aurum::Engine::Token.new('$eof', '')
+    end
+
+    def pushback symbol
+      @source.unshift(symbol)
+    end
+
+    def inspect
+      @source.inspect
+    end
+  end
+end

data/spec/pattern_matcher.rb

@@ -0,0 +1,123 @@
+module PatternMatcher
+  class BeMatchedBy
+    def initialize pattern_string, pattern
+      @pattern_string, @pattern = pattern_string, pattern
+    end
+
+    def matches?(lexeme)
+      @lexeme = lexeme
+      states = closure(@pattern.automata.table, [0])
+      lexeme.each_byte {|char| states = move(@pattern.automata.table, states, char)}
+      states.include?(@pattern.accept)
+    end
+
+    def failure_message
+      "'#@lexeme' is not matched by /#@pattern_string/"
+    end
+    
+    def negative_failure_message
+      "'#@lexeme' is matched by /#@pattern_string/"
+    end
+
+    def description
+      "match '#@lexeme' by /#@pattern_string/"
+    end
+
+    private
+    def move automata, states, char
+      result = []
+      states.each {|state| automata[state].each {|tran| result.concat(closure(automata, [tran.destination]))  if tran.character_set.include?(char)} }
+      result.uniq
+    end
+
+    def closure automata, states
+      closure, unvisited = Set.new(states.dup), states.dup
+      while !unvisited.empty?
+        automata[unvisited.pop].each do |tran|
+          [closure, unvisited].each {|x| x << tran.destination} if tran.character_set ==  Aurum::Grammar::LexicalRules::Epsilon && !closure.include?(tran.destination)
+        end
+      end
+      closure.to_a
+    end
+  end
+  def be_matched_by pattern_string, pattern
+    BeMatchedBy.new(pattern_string, pattern)
+  end
+end
+
+module DeterminizedPatternMatcher
+  class BeMatchedBy
+    def initialize pattern_string, pattern
+      @pattern_string, = pattern_string
+      @automata, @accepts =  Aurum::Grammar::LexicalRules::SubsetDeterminizer.new(pattern.automata, [pattern.accept]).determinize
+    end
+
+    def matches?(lexeme)
+      @lexeme = lexeme
+      state = 0      
+      lexeme.each_byte {|char| state = move(state, char)} rescue return false
+      @accepts.include?(state)
+    end
+
+    def failure_message
+      "'#@lexeme' is not matched by /#@pattern_string/"
+    end
+    
+    def negative_failure_message
+      "'#@lexeme' is matched by /#@pattern_string/"
+    end
+
+    def description
+      "match '#@lexeme' by /#@pattern_string/"
+    end
+
+    private
+    def move state, char      
+      @automata.table[state].each {|tran| return tran.destination if tran.character_set.include?(char)}
+      raise 'not matched'
+    end
+  end
+  def be_matched_by pattern_string, pattern
+    BeMatchedBy.new(pattern_string, pattern)
+  end
+end
+
+module MinimizedPatternMatcher
+  class BeMatchedBy
+    def initialize pattern_string, pattern
+      @pattern_string, = pattern_string
+      automata, accepts =  Aurum::Grammar::LexicalRules::SubsetDeterminizer.new(pattern.automata, [pattern.accept]).determinize
+      accept_actions = {}
+      accepts.keys.each {|state| accept_actions[state] = 'action'}
+      @automata, @accepts =  Aurum::Grammar::LexicalRules::HopcroftMinimizer.new(automata, accept_actions).minimize
+    end
+
+    def matches?(lexeme)
+      @lexeme = lexeme
+      state = 0      
+      lexeme.each_byte {|char| state = move(state, char)} rescue return false
+      @accepts.include?(state)
+    end
+
+    def failure_message
+      "'#@lexeme' is not matched by /#@pattern_string/"
+    end
+    
+    def negative_failure_message
+      "'#@lexeme' is matched by /#@pattern_string/"
+    end
+
+    def description
+      "match '#@lexeme' by /#@pattern_string/"
+    end
+
+    private
+    def move state, char      
+      @automata.table[state].each {|tran| return tran.destination if tran.character_set.include?(char)}
+      raise 'not matched'
+    end
+  end
+  def be_matched_by pattern_string, pattern
+    BeMatchedBy.new(pattern_string, pattern)
+  end  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,133 @@
+lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
+examples_dir = File.join(File.dirname(__FILE__), '..', 'examples')
+$:.unshift(lib_dir) unless $:.include?(lib_dir)
+$:.unshift(examples_dir) unless $:.include?(examples_dir)
+
+require 'delegate'
+require 'aurum/grammar'
+require 'aurum/engine'
+
+require File.join(File.dirname(__FILE__), 'pattern_matcher')
+require File.join(File.dirname(__FILE__), 'parser_matcher')
+
+class SimpleLR0Grammar < Aurum::Grammar
+  syntax_rules do
+    expression term, '+', term
+    term _id
+  end
+end
+
+class Grammar419InDragonBook < Aurum::Grammar
+  syntax_rules do
+    expression expression, '+', term
+    expression term
+    term term, '*', factor
+    term factor
+    factor '(', expression, ')'
+    factor _id    
+  end
+end
+
+class ExpressionGrammar < Aurum::Grammar
+  syntax_rules do
+    expression expression, '+', expression
+    expression expression, '*', expression
+    expression '(', expression, ')'
+    expression _id
+  end
+end
+
+class ExpressionGrammarWithOperatorPrecedence < Aurum::Grammar
+  precedences do
+    left '*'
+    left '+'
+  end
+    
+  productions do
+    expression expression, '+', expression
+    expression expression, '*', expression
+    expression '(', expression, ')'
+    expression _id
+  end
+end
+
+class BnfGrammarInCharlesThesis < Aurum::Grammar
+  productions do
+    bnf rlist
+    rlist rlist, rule
+    rlist _
+    rule _s, '->', slist
+    slist slist, _s
+    slist _
+  end
+end
+
+class Grammar411InDragonBook < Aurum::Grammar
+  productions do
+    expression  term, expression1
+    expression1 '+', term, expression1
+    expression1 _
+    term factor, term1
+    term1 '*', factor, term1
+    term1 _
+    factor '(', expression, ')'
+    factor _id
+  end
+end
+
+class GrammarWithNullableSymbols < Aurum::Grammar
+  productions do
+    expression term
+    expression factor
+    term _
+    factor _id
+  end
+end
+
+class LALR4Grammar < Aurum::Grammar
+  productions do
+    s b, _x, _x, _x, _y, _z
+    s c, _x, _x, _x, _x, _z
+    s _x, b, _x, _x, _x, _x, _y
+    b _w
+    c _w
+  end
+end
+
+class BnfGrammarWithActions < Aurum::Grammar
+  productions do
+    bnf rlist
+    rlist rlist, rule {rlist.s_exp = rlist1.s_exp + [rule.s_exp]}
+    rlist _ {rlist.s_exp = []}
+    rule _s, '->', slist {rule.s_exp = [:rule, _s.value, slist.s_exp]}
+    slist slist, _s {slist.s_exp = slist1.s_exp + [_s.value]}
+    slist _ {slist.s_exp = []}
+  end
+end
+
+class GrammarForLexerTesting < Aurum::Grammar
+  tokens do
+    _numeric range(?0, ?9).one_or_more
+    match 'state_a_begin', :shift_to => :state_a
+    match 'state_b_begin', :shift_to => :state_b, :recognize => :_state_b_begin
+    match 'state_b_beg', :recognize => :_state_b_beg
+    ignore 'ignore'
+    within :state_a, :state_b do
+      _numeric range(?0, ?9).one_or_more
+    end
+  end
+end
+
+class BNFGrammar < Aurum::Grammar
+  productions do
+    bnf bnf, rules
+    bnf _    
+    rules rules, '|', symbol_list
+    rules rule
+    rule _nonterminal, '->', symbol_list
+    symbol_list symbol_list, _nonterminal
+    symbol_list _
+  end
+end
+
+$pattern = Aurum::Grammar::LexicalRules::Pattern