aurum 0.1.1 → 0.2.0

data/lib/aurum/grammar/metalang/grammar.rb

@@ -0,0 +1,47 @@
+module Aurum
+  module MetaLanguage
+    class Grammar < Aurum::Grammar
+      tokens do
+        ignore enum(" \r\n").one_or_more
+        _nonterminal 'nonterminal'
+        _terminal    'terminal'
+        _literal     'literal'
+      end
+      
+      syntax_rules do        
+        bnf bnf, rules
+        bnf _
+        
+        rules rule
+        rules rules, '|', symbol_list
+        
+        rule _nonterminal, '->', symbol_list
+
+        symbol_list _
+        symbol_list symbol_list, symbol
+        
+        symbol _nonterminal
+        symbol _terminal   
+        symbol _literal
+        
+=begin         
+        syntax_rules syntax_rules, syntax_rule {syntax_rules.s_exp  = syntax_rules1.s_exp + syntax_rule.s_exp}
+        syntax_rules _ {syntax_rules.s_exp = []} 
+                
+#        syntax_rule _nonterminal, '->', symbols {syntax_rule.s_exp = [:production, [:nonterminal, _nonterminal.value], symbols.s_exp]}
+        syntax_rule _nonterminal, '->', handles {syntax_rule.s_exp = [:production, [:nonterminal, _nonterminal.value], handles.s_exp]}
+        
+        handles handle, '|', handles
+        handles _
+                        
+        handle handle, symbol {handle.s_exp = handle1.s_exp << symbol.s_exp}
+        handle _           {handle.s_exp = []}
+        
+        symbol _nonterminal {symbol.s_exp = [:nonterminal, _nonterminal.value]}
+        symbol _terminal    {symbol.s_exp = [:terminal, _terminal.value]}
+        symbol _literal     {symbol.s_exp = [:literal, _literal.value]}
+=end        
+      end
+    end
+  end
+end

data/lib/aurum/grammar/syntax_rules.rb

@@ -0,0 +1,95 @@
+module Aurum
+  class Grammar
+    class SyntaxRules < Struct.new(:precedences, :productions)
+      attr_reader :literals
+      def initialize
+        @precedences, @productions, @terminals, @literals = {}, {}, {}, [].to_set
+        @precedences_update = true
+      end
+    
+      def add_syntax_rule production
+        @precedences_update = true
+        nonterminal = production.nonterminal
+        production.symbols.each {|s| @terminals[s.name] = s if s.is_terminal && !s.precedence}
+        @productions[nonterminal] = [] unless @productions.has_key?(nonterminal)
+        return false if @productions[nonterminal].include?(production)
+        @productions[nonterminal] << production
+        true
+      end
+      
+      def add_literal literal
+        @literals << literal
+      end
+                  
+      def add_operator_precedence operator, precedence
+        @precedences_update = true
+        @precedences[operator] = precedence
+      end
+    
+      def productions nonterminal
+        @productions[nonterminal]
+      end
+      
+      def operator_precedence symbol_name
+        @precedences[symbol_name]
+      end
+      
+      def assign_operator_precedence_to_symbols
+        return unless @precedences_update
+        @precedences_update = false
+        for name, precedence in @precedences
+          symbol = @terminals[name]
+          symbol.precedence = precedence if symbol
+        end
+      end
+            
+      class Precedence < Struct.new(:associativity, :level)      
+        include Comparable
+        Highest, Lowest = Precedence.new(:non_associative, 65535), Precedence.new(:non_associative, -65535)
+        def <=> other
+          return level <=> other.level unless level == other.level
+          return 1 if associativity == :left_associative
+          return -1 if associativity == :right_associative
+          #TODO non_associative
+        end
+      end
+    
+      class Symbol < Struct.new(:name, :is_terminal)
+        attr_accessor :precedence, :action
+        def inspect
+          name.split('$literal_').last
+        end
+      end
+    
+      class Production < Struct.new(:nonterminal, :symbols)
+        attr_accessor :name
+        
+        def precedence
+          terminals = symbols.find_all {|symbol| symbol.is_terminal && symbol.precedence}
+          terminals.size == 1 ? terminals.first.precedence : Precedence::Highest
+        end
+        
+        def inspect
+          "#{nonterminal.inspect} -> #{symbols.map {|symbol| symbol.inspect}.join(' ')}"
+        end
+      end
+    end
+  
+    def Grammar.terminal name
+      SyntaxRules::Symbol.new(name, true)
+    end
+  
+    def Grammar.nonterminal name
+      SyntaxRules::Symbol.new(name, false)
+    end
+    
+    def Grammar.precedence associative, level
+      SyntaxRules::Precedence.new(associative, level)
+    end
+    
+    def Grammar.production nonterminal, symbols
+      SyntaxRules::Production.new(nonterminal, symbols)
+    end
+    EOF, Epsilon = Grammar.terminal('$eof'), Grammar.terminal('$epsilon')
+  end
+end

data/spec/builder/dsl_definition/aurum_grammar_spec.rb

@@ -0,0 +1,33 @@
+helper_lib = "#{File.dirname(__FILE__)}/../../"
+$:.unshift helper_lib unless $:.include? helper_lib
+#require 'spec_helper'
+#require File.join(File.dirname(__FILE__), '../../engine/parser_matcher')
+=begin
+class TestGrammar < Aurum::Grammar
+end
+class GrammarA < Aurum::Grammar
+  productions do
+    expression factor, '+', factor
+    factor _id
+  end
+end
+
+describe Aurum::Grammar, 'including other grammar' do
+  def be_parsed_as(nonterminal, value=nil)    
+    ParserMatcher::BeParsedAs.new(TestGrammar.parsing_table(nonterminal), nonterminal, value)
+  end
+
+  it 'should throw an error if included object is not a grammar' do
+    lambda {TestGrammar.include_grammar String}.should raise_error(RuntimeError, 'String is not a grammar!')
+  end
+  
+  it 'should include grammar into TestGrammar' do
+    TestGrammar.include_grammar GrammarA
+    [:_id, '+', :_id].should be_parsed_as('expression')
+  end
+  it 'should change symbol names and include grammar into TestGrammar' do
+    TestGrammar.include_grammar GrammarA, :syntax => {:expression => :exp}
+    [:_id, '+', :_id].should be_parsed_as('exp')    
+  end
+end
+=end

data/spec/engine/lexer_spec.rb

@@ -0,0 +1,59 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Engine::BasicTokenizationCapability do
+  before :all do
+    Lexer = Aurum::Lexer.new(GrammarForLexerTesting)
+  end
+  
+  it 'should return recognized symbol' do
+    lexer = Lexer.new('1234')
+    lexer.next_symbol.to_a.should == ['_numeric', '1234']
+  end
+
+  it 'should shift to lexical state' do
+    lexer = Lexer.new('state_a_begin1234')
+    lexer.next_symbol.to_a.should == ['_numeric', 'state_a_begin1234']
+  end
+
+  it 'should return recognized symbol and shift to lexical state' do
+    lexer = Lexer.new('state_b_begin1234')
+    lexer.next_symbol.to_a.should == ['_state_b_begin', 'state_b_begin']
+    lexer.next_symbol.to_a.should == ['_numeric', '1234']
+  end
+  
+  it 'should ignore ignored pattern' do
+    lexer = Lexer.new('ignore1234')
+    lexer.next_symbol.to_a.should == ['_numeric', '1234']
+  end
+  
+  it 'should not match pattern greedily' do
+    lexer = Lexer.new('state_b_begi1234')
+    lexer.next_symbol.to_a.should == ['_state_b_beg', 'state_b_beg']
+  end
+  
+  it 'should return EOF if reach the end of input' do
+    lexer = Lexer.new('ignore')
+    lexer.next_symbol.to_a.should == ['$eof', '']
+  end
+  
+  it 'should return UNKNOWN if can not recognize token' do
+    lexer = Lexer.new('state_b_')
+    lexer.next_symbol.to_a.should == ['$unknown', 'state_b_']
+  end
+  
+  it 'should return pushbacked symbol' do
+    lexer = Lexer.new('state_b_')
+    lexer.pushback('pushback')
+    lexer.next_symbol.should == 'pushback'
+  end
+  
+  it 'should return current column' do
+    lexer = Lexer.new('1234')
+    lexer.column.should == 0
+    lexer.next_symbol
+    lexer.column.should == 4
+  end  
+end

data/spec/engine/parser_spec.rb

@@ -0,0 +1,90 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+
+describe Aurum::Engine::BasicParsingCapability, ' LR(0) : simple LR(0) grammar' do
+  include ParserMatcher
+  before :all do
+    @grammar = SimpleLR0Grammar
+  end
+  it {[:_id].should be_parsed_as('term')}
+  it {[:_id, '+', :_id].should be_parsed_as('expression')}
+  it {[:_it].should_not be_parsed_as('term')}
+  it {[:_id, '+', :_it].should_not be_parsed_as('expression')}
+  it 'should not look for eof' do
+    [:_id, '+', :_id].should be_parsed_as('term')
+  end
+end
+
+describe Aurum::Engine::BasicParsingCapability, ' LALR(1) : Grammar 4.19 in Dragon book' do
+  include ParserMatcher
+  before :all do
+    @grammar = Grammar419InDragonBook
+  end
+  it {[:_id].should be_parsed_as('factor')}
+  it {['(', :_id, ')'].should be_parsed_as('factor')}
+  it {['(', :_id, '*', :_id, ')'].should be_parsed_as('factor')}
+  it {['(', :_id, '+', :_id, ')'].should be_parsed_as('factor')}
+  it {[:_id, '*', :_id].should be_parsed_as('term')}
+  it {[:_id, '*', '(', :_id, '*', :_id, ')'].should be_parsed_as('term')}
+  it {[:_id, '*', '(', :_id, '+', :_id, ')'].should be_parsed_as('term')}
+  it {[:_id, '+', :_id].should be_parsed_as('expression')}
+  it {[:_id, '*', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', :_id, '*', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', :_id, '+', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', '(', :_id, '*', :_id, ')'].should be_parsed_as('expression')}
+  it {[:_id, '+', '(', :_id, '+', :_id, ')'].should be_parsed_as('expression')}
+  it {[:_id, '+', :_id].should_not be_parsed_as('term')}
+end
+
+describe Aurum::Engine::BasicParsingCapability, ' LALR(1) : Expression Grammar with operator precedences' do
+  include ParserMatcher
+  before :all do
+    @grammar = ExpressionGrammarWithOperatorPrecedence
+  end
+  it {[:_id, '+', :_id].should be_parsed_as('expression')}
+  it {[:_id, '*', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', :_id, '*', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', :_id, '+', :_id].should be_parsed_as('expression')}
+  it {[:_id, '+', '(', :_id, '*', :_id, ')'].should be_parsed_as('expression')}
+  it {[:_id, '+', '(', :_id, '+', :_id, ')'].should be_parsed_as('expression')}
+end
+
+describe Aurum::Engine::SemanticActionExecutable, ' LALR(2) : BNF grammar in Philippe Charles\' thesis' do
+  include ParserMatcher
+  before :all do
+    @grammar = BnfGrammarWithActions
+  end
+  it {[:_s, '->', :_s, :_s, :_s].should be_parsed_as('bnf', :s_exp => [[:rule, :_s, [:_s, :_s, :_s]]])}
+  it {[:_s, '->', :_s, :_s, '->', :_s].should be_parsed_as('bnf', :s_exp => [[:rule, :_s, [:_s]], [:rule, :_s, [:_s]]])}
+  it {[:_s, '->', :_s, '->', :_s].should be_parsed_as('bnf', :s_exp => [[:rule, :_s, []], [:rule, :_s, [:_s]]])}
+  it {[:_s, '->', :_s, '->'].should be_parsed_as('bnf', :s_exp => [[:rule, :_s, []], [:rule, :_s, []]])}
+end
+
+describe Aurum::Engine::BasicParsingCapability, ' LALR(4) : simple LALR(4) grammar' do
+  include ParserMatcher
+  before :all do
+    @grammar = LALR4Grammar
+  end  
+  it {[:_x, :_w, :_x, :_x, :_x, :_x, :_y].should be_parsed_as('s')}
+  it {[:_w, :_x, :_x, :_x, :_y, :_z].should be_parsed_as('s')}
+  it {[:_w, :_x, :_x, :_x, :_x, :_z].should be_parsed_as('s')}
+end
+
+describe Aurum::Engine::SemanticActionExecutable::SemanticContext do
+  it 'lexer should be accessible in semantic context' do
+    @context = Aurum::Engine::SemanticActionExecutable::SemanticContext.new('lexer', {}, {})
+    (@context.instance_eval {@lexer}).should == 'lexer'
+  end
+  
+  it 'should access symbols by name' do
+    @context = Aurum::Engine::SemanticActionExecutable::SemanticContext.new('lexer', {'symbolA' => 1}, {})
+    @context.symbolA.should == 1
+  end
+  
+  it 'should access symbols by alias name' do
+    @context = Aurum::Engine::SemanticActionExecutable::SemanticContext.new('lexer', {'symbolA' => 1}, {'symbolB' => 'symbolA'})
+    @context.symbolB.should == 1
+  end
+end

data/spec/examples/dangling_else_example.rb

@@ -0,0 +1,30 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+require 'dangling_else/grammar'
+
+describe 'Dangling Else' do
+  before :all do
+    DanglingElseLexer = Aurum::Lexer.new(Aurum::Examples::DanglingElse)
+    DanglingElseParser = Aurum::Parser.new(Aurum::Examples::DanglingElse, :statement)
+  end
+  
+  it 'else should be matched to closest if' do
+    parse(<<-EOF).should == [:if, [:expr], [:if, [:expr], [:other], [:other]], nil]
+    if ( expr ) 
+       if (expr)
+          other
+       else
+          other
+    EOF
+  end
+    
+  def scanner source
+    DanglingElseLexer.new(source)
+  end
+  
+  def parse source
+    DanglingElseParser.new.parse(DanglingElseLexer.new(source)).s_exp
+  end
+end

data/spec/examples/expression_example.rb

@@ -0,0 +1,48 @@
+helper_dir = File.join(File.dirname(__FILE__), '..')
+$:.unshift(helper_dir) unless $:.include?(helper_dir)
+
+require 'spec_helper'
+require 'expression/grammar'
+
+describe 'Expression Grammar' do
+  before :all do
+    ExpressionLexer = Aurum::Lexer.new(Aurum::Examples::ExpressionGrammar)
+    ExpressionParser = Aurum::Parser.new(Aurum::Examples::ExpressionGrammar, :expression)
+  end
+  
+  it '1 + 1 should == 2' do
+    parse('1 + 1').should == 2
+  end
+
+  it '2 * 3 should == 6' do
+    parse('2 * 3').should == 6
+  end
+
+  it '2 + 2 * 3 should == 8' do
+    parse('2 + 2 * 3').should == 8
+  end
+
+  it '(2 + 2) * 3 should == 12' do
+    parse('(2 + 2) * 3').should == 12
+  end
+  
+  it '3 * (2 + 2) should == 12' do
+    parse('3 * (2 + 2)').should == 12
+  end  
+
+  it '2 * (2 + 2) * 3 should == 24' do
+    parse('2 * (2 + 2) * 3').should == 24
+  end
+  
+  it '-2 * -4 should == 8' do
+    parse('-2 * -4').should == 8
+  end
+  
+  it '-2 - - 4 should == 2' do
+    parse('-2 - - 4').should == 2
+  end
+
+  def parse source
+    ExpressionParser.new.parse(ExpressionLexer.new(source)).value
+  end
+end

data/spec/examples/smalltalk_example.rb

@@ -0,0 +1,50 @@
+=begin
+helper_dir = "#{File.dirname(__FILE__)}/../"
+$:.unshift helper_dir unless $:.include?(helper_dir)
+require 'spec_helper'
+require 'smalltalk/grammar'
+
+describe 'Little Smalltalk' do
+  before :all do
+    @logger = Aurum::Builder::GraphvizLogger.new
+    @parsing_table = Smalltalk::Grammar.parsing_table :program, @logger
+    @lexical_table = Smalltalk::Grammar.lexical_table @logger
+  end
+
+  it 'keyword message sending should be parsed correctly' do
+    parse(<<-EOF).should == [:program, [], [[:keyword, [:var, "Transcript"], [["show:", [:var, "helloMessage"]]]]]]
+      Transcript show: helloMessage.
+    EOF
+  end
+
+  it 'binary message sending should be parsed correctly' do
+    parse(<<-EOF).should == [:program, [], [[:binary, [:var, "a"], [["+", [:var, "b"]]]]]]
+      a + b.
+    EOF
+  end
+
+  it 'unary message sending should be parsed correctly' do
+    parse(<<-EOF).should == [:program, [], [[:unary, [:var, "car"], ["go"]]]]
+      car go.
+    EOF
+  end
+
+  it 'unary message sending should have higher precedence then keyword message sending' do
+    parse(<<-EOF).should == [:program, [], [[:keyword, [:var, "Rectangle"], [["width:", [:unary, [:var, "aRect"], ["weight"]]], ["height:", [:unary, [:var, "aRect"], ["height"]]]]]]]
+      Rectangle width:aRect weight height:aRect height.
+    EOF
+  end
+
+  it 'binary message sending should have higher precedence then keyword message sending' do
+    parse(<<-EOF).should == [:program, [], [[:keyword, [:var, "Rectangle"], [["width:", [:binary, [:var, "width"], [["+", [:var, "delta"]]]]], ["height:", [:binary, [:var, "height"], [["+", [:var, "delta"]]]]]]]]]
+      Rectangle width:width + delta height:height + delta.
+    EOF
+  end
+
+  def parse source
+    lexer = Aurum::Engine::Lexer.new(@lexical_table, source)
+    parser = Aurum::Engine::Parser.new(@parsing_table)
+    parser.parse(lexer).s_exp
+  end
+end
+=end