aurum 0.1.1 → 0.2.0

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/gempackagetask'
+require 'spec/rake/spectask'
+
+spec = Gem::Specification.new do |s|
+    s.name       = 'aurum'
+    s.version    = '0.2.0'
+    s.author     = 'Vincent Xu'
+    s.email      = 'x at bjug dot org'
+    s.homepage   = 'http://rubyforge.org/projects/aurum'
+    s.platform   = Gem::Platform::RUBY
+    s.summary    = 'Aurum is a LALR(n) parser generator written in Ruby.'
+    s.files      = FileList['{examples,lib,spec}/**/*', 'Rakefile'].to_a
+    s.require_path      = 'lib'
+    s.autorequire       = 'aurum'
+    s.has_rdoc          = false
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+end
+
+Spec::Rake::SpecTask.new('spec') do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb', 'spec/**/*_example.rb']
+  t.spec_opts = ['--format', 'html:result.html', '--format', 'specdoc']
+end
+
+task :default => [:spec]

data/examples/dangling_else/grammar.rb

@@ -0,0 +1,23 @@
+require 'aurum'
+module Aurum
+  module Examples
+class DanglingElse < Aurum::Grammar
+  tokens do
+    ignore enum(" \r\n").one_or_more
+  end
+  
+  productions do    
+    statement 'if', '(', expression, ')', restricted_statement, 
+      'else', statement {statement.s_exp = [:if, expression.s_exp, restricted_statement.s_exp, statement1.s_exp]}
+    statement 'if', '(', expression, ')', 
+      statement {statement.s_exp = [:if, expression.s_exp, statement1.s_exp, nil]}
+    statement simple_statement
+    restricted_statement 'if', '(', expression, ')', restricted_statement, 
+      'else', restricted_statement {statement.s_exp = [:if, expression.s_exp, restricted_statement1.s_exp, restricted_statement2.s_exp]}
+    restricted_statement simple_statement
+    simple_statement 'other' do simple_statement.s_exp = [:other] end
+    expression 'expr'        do expression.s_exp = [:expr] end
+  end
+end
+end
+end

data/examples/expression/grammar.rb

@@ -0,0 +1,28 @@
+require 'aurum'
+module Aurum
+  module Examples
+    class ExpressionGrammar < Aurum::Grammar
+      tokens do
+        ignore string(' ').one_or_more
+        _number range(?0, ?9).one_or_more
+      end
+
+      precedences do
+        left '*', '/'
+        left '+', '-'
+      end
+
+      productions do
+        expression expression, '+', expression {expression.value = expression1.value + expression2.value}
+        expression expression, '-', expression {expression.value = expression1.value - expression2.value}
+        expression expression, '*', expression {expression.value = expression1.value * expression2.value}
+        expression expression, '/', expression {expression.value = expression1.value / expression2.value}
+        expression '(', expression, ')'        do expression.value = expression1.value end
+        expression _number                     {expression.value = _number.value.to_i}
+        expression '+', _number                {expression.value = _number.value.to_i}
+        expression '-', _number                {expression.value = -_number.value.to_i}
+      end
+    end
+  end
+end
+

data/examples/smalltalk/grammar.rb

@@ -0,0 +1,151 @@
+require 'aurum'
+=begin
+module Smalltalk
+  class UnaryMessageSending < Aurum::Grammar
+    productions do
+      unary_expression higher_precedence, unary_messages do
+        unary_expression.s_exp = [:unary, higher_precedence.s_exp, unary_messages.s_exp]
+      end    
+      unary_messages unary_messages, _message do
+        unary_messages.s_exp = unary_messages1.s_exp + [_message.value]
+      end
+      unary_messages _message do
+        unary_messages.s_exp = [_message.value]
+      end
+    end
+  end
+  
+  class BinaryMessageSending < Aurum::Grammar
+    productions do
+      binary_expression higher_precedence, binary_messages do
+        binary_expression.s_exp = [:binary, higher_precedence.s_exp, binary_messages.s_exp]
+      end
+      binary_messages binary_messages, binary_message do
+        binary_messages.s_exp << binary_message.s_exp
+      end
+      binary_messages binary_message
+      binary_message _binary_message, higher_precedence do
+        binary_message.s_exp = [[_binary_message.value, higher_precedence.s_exp]]
+      end
+    end
+  end
+  
+  class KeywordMessageSending < Aurum::Grammar
+    productions do
+      keyword_expression higher_precedence, keyword_messages do
+        keyword_expression.s_exp = [:keyword, higher_precedence.s_exp, keyword_messages.s_exp]
+      end
+      keyword_messages keyword_messages, keyword_message do
+        keyword_messages.s_exp = keyword_messages1.s_exp +  keyword_message.s_exp
+      end
+      keyword_messages keyword_message
+      keyword_message _keyword, higher_precedence do
+        keyword_message.s_exp = [[_keyword.value, higher_precedence.s_exp]]
+      end
+    end
+  end
+  
+  class Block < Aurum::Grammar
+    productions do
+      block '[', block_parameters, '|', sentences, ']' do 
+        block.s_exp = [:block, block_parameters.s_exp, sentences.s_exp] 
+      end
+      block '[', sentences, ']' do 
+        block.s_exp = [:block, [:params], sentences.s_exp] 
+      end
+      block_parameters block_parameters, ':', _identifier do 
+        block_parameters.s_exp = block_parameters1.s_exp + _identifier.value
+      end
+      block_parameters ':', _identifier do       
+        block_parameters.s_exp = [:params, _identifier.value]
+      end
+    end
+  end
+  
+  class Grammar < Aurum::Grammar
+    character_classes do
+      letter range(?a, ?z) + range(?A, ?Z)
+      identifier_tail letter + range(?0, ?9) + enum('_')
+    end
+  
+    tokens do
+      comment double_quote, ~double_quote
+      whitespaces enum(" \r\n").one_or_more
+
+      identifier letter, identifier_tail.zero_or_more
+    
+      _identifier identifier
+      _binary_selector enum('!%&*+/<=>?@~,\\').one_or_more
+      _keyword identifier, ':'
+      ignore comment
+      ignore whitespaces
+    end
+    
+    include_grammar UnaryMessageSending,  :syntax => {:higher_precedence => :primary, :_message => :_identifier}
+    include_grammar BinaryMessageSending, :syntax => {:higher_precedence => :higher_precedence_binary, :_binary_message => :_binary_selector}
+    include_grammar KeywordMessageSending,:syntax => {:higher_precedence => :higher_precedence_keyword}
+    include_grammar Block
+  
+    productions do
+      program variables, sentences do      
+        program.s_exp = [:program, variables.s_exp, sentences.s_exp]
+      end
+
+      variables '|', identifiers, '|' do
+        variables.s_exp = [] + identifiers.s_exp
+      end
+      variables _ do
+        variables.s_exp = []
+      end
+
+      identifiers identifiers, _identifier do
+        identifiers.s_exp = identifiers.s_exp + _identifier.value
+      end
+      identifiers _ do
+        identifiers.s_exp = []
+      end
+
+      sentences sentences, expression, '.' do
+        sentences.s_exp = sentences1.s_exp + [expression.s_exp]
+      end
+      sentences _ do
+        sentences.s_exp = []
+      end   
+      sentence expression, '.' do
+        sentence.s_exp = expression.s_exp
+      end
+    
+      expression _identifier, ':=', message_expression do
+        expression.s_exp = [:assign, _identifier.value, message_expression.s_exp]
+      end
+      expression _identifier, ':=', primary do
+        expression.s_exp = [:assign, _identifier.value, primary.s_exp]
+      end
+      expression message_expression
+      expression primary
+
+      primary _identifier do
+        primary.s_exp = [:var, _identifier.value]
+      end
+      primary block 
+      primary '(', message_expression, ')' do
+        primary.s_exp = message_expression.s_exp
+      end
+      primary '(', primary, ')' do
+        primary.s_exp = primary1.s_exp
+      end
+
+      message_expression unary_expression
+      message_expression binary_expression
+      message_expression keyword_expression
+      
+      higher_precedence_binary primary
+      higher_precedence_binary unary_expression
+      
+      higher_precedence_keyword primary
+      higher_precedence_keyword unary_expression
+      higher_precedence_keyword binary_expression      
+    end
+  end  
+end
+=end

data/examples/smalltalk/interpreter.rb

@@ -0,0 +1,70 @@
+require File.join(File.dirname(__FILE__), 'grammar')
+require 'stringio'
+=begin
+class Smalltalk::Interpreter
+  @@parsing_table = Smalltalk::Grammar.parsing_table :program
+  @@lexical_table = Smalltalk::Grammar.lexical_table
+  
+  def initialize context = {}
+    @parser = Aurum::Engine::Parser.new(@@parsing_table)
+    @context = context
+  end
+  
+  def run source
+    @lexer = Aurum::Engine::Lexer.new(@@lexical_table, source)
+    eval_s_exp(@parser.parse(@lexer).s_exp)
+  end
+  
+  private
+  def program variables, sentences
+    variables.each {|variable| @context[variable] = nil}
+    sentences.each {|sentence| eval_s_exp(sentence)}
+  end
+    
+  def var name
+    @context[name]
+  end
+  
+  def unary object_s_exp, unary_messages
+    unary_messages.inject((eval_s_exp(object_s_exp))) {|obj, message| obj = obj.send(message)}
+  end
+  
+  def binary object_s_exp, binary_messages
+    binary_messages.inject((eval_s_exp(object_s_exp))) {|obj, message| obj = obj.send(message.first, eval_s_exp(message.last))}
+  end
+  
+  def keyword object_s_exp, keyword_messages
+    obj = eval_s_exp(object_s_exp)
+    messages, values = '', []
+    for keyword_message in keyword_messages
+      messages << keyword_message.first
+      values << eval_s_exp(keyword_message.last)
+    end
+    obj.send(messages.gsub(':', '_'), values)
+  end
+  
+  def eval_s_exp s_exp
+    self.send(s_exp.first, *s_exp[1..-1])
+  end
+end
+
+class Transcript
+  def self.show_ message
+    puts message
+  end
+  
+  def self.hello
+    show_ 'Hello'
+  end
+end
+
+interpreter = Smalltalk::Interpreter.new 'Transcript' => Transcript, 
+                                         'helloMessage' => 'Hello World'
+                                       
+interpreter.run(<<-EOF)
+  "keyword message sending"
+  Transcript show: helloMessage.
+  "unary message sending"
+  Transcript hello.
+EOF
+=end

data/examples/yacc/grammar.rb

@@ -0,0 +1,72 @@
+require 'aurum'
+module Aurum
+  module Examples
+    class YaccGrammar < Aurum::Grammar
+      character_classes do
+        letter range(?a, ?z) + range(?A, ?Z)
+        id_tail range(?a, ?z) + range(?A, ?Z) + range(?0, ?9) + string('_')
+        comment_char any - enum('*')
+      end
+      
+      tokens do
+        multi_lines_comment '/*', ~ string('*/')
+        single_line_comment '//', ~ (string("\r") | string("\n"))        
+        
+        _identifier letter, id_tail.zero_or_more
+        _literal "'", ~ string("'")
+        _declaration_code '%{', ~ string('%}')
+        _union_name '<', ~ string('>')
+        _source_code '{', ~ string('}')
+        
+        
+        ignore enum(" \r\n\t").one_or_more 
+        ignore multi_lines_comment
+        ignore single_line_comment
+      end
+      
+      productions do 
+        grammar tokens, '%%', rules, end_marker
+
+        tokens tokens, token
+        tokens _
+
+        token '%start', _identifier
+        token '%union', _source_code
+        token '%token', union_name, symbols
+        token '%left', union_name, symbols
+        token '%right', union_name, symbols
+        token '%nonassoc', union_name, symbols
+        token '%type', union_name, symbols
+        token _declaration_code
+
+        symbols symbols, symbol
+        symbols symbol
+
+        symbol _identifier
+        symbol _literal
+
+        end_marker '%%'
+        end_marker _
+
+        union_name _union_name
+        union_name _
+
+        rules rules, rule
+        rules _
+
+        rule _identifier, ':', rule_body, ';'
+        rule_body rule_body, '|', rule_handle
+        rule_body rule_handle
+
+        rule_handle symbols, prec, source_code
+        rule_handle source_code
+
+        prec '%prec', symbol
+        prec _
+
+        source_code _source_code
+        source_code _
+      end
+    end
+  end
+end

data/lib/aurum.rb

@@ -1,10 +1,2 @@
-require 'aurum/lexical_table_generator'
-require 'aurum/parsing_table_generator'
-require 'aurum/engine'
 require 'aurum/grammar'
-
-Enumerable.class_eval do
-    def grep_each condition
-        condition.kind_of?(Proc) ? each {|x| yield x if condition.call x} : each {|x| yield x if eval(condition.to_s)}
-    end
-end
+require 'aurum/engine'

data/lib/aurum/engine.rb

@@ -1,175 +1,39 @@
-module Aurum		
-	IO.class_eval do
-		def get_char
-			char, @pushback_char = @pushback_char ? @pushback_char : self.getc, nil			
-			char > 128 ? char * 128 + self.getc : char 
-		end
-		
-		def pushback char
-			@pushback_char = char
-		end
-	end
-	
-	String.class_eval do
-		def get_char
-			@get_char_index = -1 unless @get_char_index
-		    self[@get_char_index += 1] 
-		end
-		
-		def eof?
-			@get_char_index = -1 unless @get_char_index
-			@get_char_index >= (self.length - 1)
-		end
-		
-		def pushback char
-			@get_char_index -= 1
-		end		
-	end
-	
-	Symbol.class_eval {attr_accessor :value}
-	Unknown = Symbol.new '$unknown', true
-	class Lexer
-		def initialize table, accepts, lexical_states, input
-			@table, @accepts, @lexical_states, @input  = table, accepts, lexical_states, input			 
-			shift_to :initial 
-		end
-		
-		def next_symbol
-			return Aurum::EOF if @input.eof?
-			@recognized, lexeme, @pushback_symbol = @pushback_symbol, '', nil
-			until @recognized
-				next_state, char = @start_state, nil
-				while next_state
-					lexeme << char if char
-					state, char = next_state, @input.get_char
-					next_state = goto state, char 		
-				end				
-				@input.pushback char
-				return Unknown unless actions = @accepts[state]
-				if actions.first == IgnoreAction
-					lexeme = ''
-				else
-					actions.first.execute self, lexeme
-				end					 				
-			end
-			@recognized.value = lexeme unless @recognized.value 
-			@recognized			
-		end
-		
-		def pushback symbol
-			@pushback_symbol = symbol
-		end
-		
-		def goto state, input
-			return nil unless input
-			next_state = @table[state].find {|tran| tran.symbols.include?(input)} 
-			next_state ? next_state.destination : nil
-		end
-		
-		def shift_to state
-			@start_state = goto 0, -@lexical_states.index(state)-1
-		end
-		
-		def recognize token
-			@recognized = Aurum::Symbol.new token, true
-		end		
-	end		
-	
-	RecognizeTokenAction.class_eval do
-		def execute lexer, lexeme
-			recognized = lexer.recognize token
-			action.call recognized, lexeme if action
-		end
-	end
-	
-	ChangeStateAction.class_eval do
-		def execute lexer, lexeme
-			lexer.shift_to state
-		end
-	end
-	
-	RecognizeTokenAndChangeStateAction.class_eval do
-		def execute lexer, lexeme
-			lexer.recognize token
-			lexer.shift_to state
-		end
-	end
-	
-	class Parser	
-		def initialize productions, parsing_table
-			@productions, @parsing_table = productions, parsing_table
-		end
-		
-		def parse lexer
-			lookahead, state_stack, symbol_stack, value_stack = lexer.next_symbol, [0], [], []
-			lookahead_shift = 0
-			while (true)
-				state = @parsing_table[state_stack.last]
-				action = state[lookahead]
-				if action.kind_of? ShiftAction
-					state_stack.push action.state
-					symbol_stack.push lookahead unless action.is_lookahead_shift
-					action.is_lookahead_shift ? lookahead_shift += 1 : lookahead = lexer.next_symbol  
-				elsif action.kind_of? ReduceAction
-					handle = @productions[action.handle]
-					lookahead_shift.times { state_stack.pop }					
-					lookahead_shift = 0
-					if action.is_read_reduce
-						state_stack.push state
-						symbol_stack.push lookahead
-						lookahead = lexer.next_symbol
-					end
-					state_stack.slice! -handle.symbols.length..-1
-					symbols = symbol_stack.slice! -handle.symbols.length..-1					
-					handle.nonterminal == Aurum::START and return value_stack.pop
-					if handle.action
-						context = {handle.nonterminal.name => [SemanticAttributes.new]}
-						handle.symbols.reverse.each_with_index do |symbol, index|
-							context[symbol.name] = [] unless context.has_key? symbol.name
-							context[symbol.name] << (symbol.is_terminal ? symbols[-index-1] : value_stack.pop)
-						end
-						SemanticActionContext.new(context).instance_eval &handle.action
-						value_stack.push context[handle.nonterminal.name][0] if context[handle.nonterminal.name]
-					end
-					goto = @parsing_table[state_stack.last][handle.nonterminal]
-					if goto.kind_of? ShiftAction
-						state_stack.push goto.state
-						symbol_stack.push nil
-					else
-						lexer.pushback lookahead
-						lookahead = handle.nonterminal
-					end
-				else
-					error_recover
-				end
-			end
-		end
-		
-		class SemanticActionContext
-			instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
-			def initialize hash
-				@hash = hash
-			end
-		
-			def method_missing name, *args
-				name_string = name.to_s
-				index = name_string =~ /\d+/ ? name_string.slice!(/\d+/).to_i : 0
-				@hash[name_string][-index] and return @hash[name_string][-index]
-				SemanticAttributes.new
-			end
-		end
-		
-		class SemanticAttributes
-			instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new'}
-			def initialize
-				@hash = {}
-			end
-		
-			def method_missing name, *args
-				name_string = name.to_s
-				return @hash[name_string] unless name_string[-1] == 61
-				@hash[name_string.slice(0..-2)] = args.first
-			end
-		end	
-	end	
-end
+require File.join(File.dirname(__FILE__), 'engine/parsing_facility')
+require File.join(File.dirname(__FILE__), 'engine/tokenization_facility.rb')
+
+module Aurum
+  class Parser
+    def Parser.new grammar, start_symbol
+      Class.new do
+        @@parsing_table, @@semantic_actions = grammar.start_from(start_symbol)
+        include Aurum::Engine::BasicParsingCapability
+        include Aurum::Engine::SemanticActionExecutable
+        private
+        def parsing_table
+          @@parsing_table
+        end        
+        def semantic_actions
+          @@semantic_actions
+        end
+      end
+    end
+  end
+  
+  class Lexer
+    def Lexer.new grammar
+      Class.new do
+        @@lexical_table = grammar.lexical_table
+        include Aurum::Engine::BasicTokenizationCapability
+        def initialize input
+          @input = Aurum::Engine::PushbackString.new(input)
+          @states, @line, @column, @push_back = [], 0, 0, []
+          shift_to('initial')
+        end
+        private
+        def lexical_table
+          @@lexical_table
+        end
+      end
+    end
+  end
+end