aurum 0.1.1 → 0.2.0

data/lib/aurum/engine/parsing_facility.rb

@@ -0,0 +1,107 @@
+module Aurum
+  module Engine
+    module BasicParsingCapability
+      def parse lexer
+        @lookahead, @lookaheads = lexer.next_symbol, []
+        @state_stack, @value_stack = [0], []
+        loop do
+          action = parsing_table.actions[current_state][@lookahead.to_s]
+          error_recovery unless action
+          if action.shift_action?
+            @state_stack.push(action.value)
+            if action.lookahead_shift?
+              @lookaheads << @lookahead
+            else
+              @value_stack.push(@lookahead) unless @lookahead.is_a?(String)
+            end
+            @lookahead = lexer.next_symbol
+          elsif action.reduce_action?
+            pushback_lookaheads(lexer) unless @lookaheads.empty?
+            prepare_read_reduce(lexer) if action.read_reduce?
+            handle = action.value
+            return @value_stack.pop if handle.nonterminal == Aurum::Builder::StartSymbol
+            nonterminal = reduce_to(handle, lexer)
+            goto = parsing_table.actions[current_state][nonterminal]
+            if goto.shift_action?
+              @state_stack.push(goto.value)
+            elsif goto.reduce_action?          
+              lexer.pushback(@lookahead)
+              @lookahead = nonterminal
+            end
+          end
+        end
+      end
+
+      private
+      def current_state
+        @state_stack.last
+      end
+
+      def pushback_lookaheads lexer
+        @state_stack.slice!(-@lookaheads.size, @lookaheads.size)
+        lexer.pushback(@lookahead)
+        @lookahead = @lookaheads.shift
+        lexer.pushback(@lookaheads.pop) until @lookaheads.empty?
+      end
+
+      def prepare_read_reduce lexer
+        @state_stack.push(:placeholder)
+        @value_stack.push(@lookahead) unless @lookahead.is_a?(String)        
+        @lookahead = lexer.next_symbol
+      end
+      
+      def reduce_to handle, lexer
+        nonterminal, length = handle.nonterminal, handle.symbols.length
+        @state_stack.slice!(-length, length)
+        if semantic_actions.has_key?(handle.name)
+          @value_stack.push(execute_semantic_action(lexer, handle, @value_stack.slice!(-length, length)))
+        end
+        nonterminal.name rescue nonterminal
+      end
+    end
+    
+    module SemanticActionExecutable
+      def execute_semantic_action lexer, handle, values
+        nonterminal_name = handle.nonterminal.name rescue handle.nonterminal
+        context = {nonterminal_name => [Attributes.new]}
+        handle.symbols.reverse.each_with_index do |symbol, index|
+          name = symbol.name rescue symbol
+          context[name] = [] unless context.has_key?(name)
+          context[name] << values[-index-1]
+        end
+        mappings = (handle.mappings || {} rescue {})
+        action = semantic_actions[handle.name]
+        SemanticContext.new(@lexer, context, mappings).instance_eval(&action)
+        context[nonterminal_name] ? context[nonterminal_name][0] : nil
+      end
+
+      class SemanticContext
+        instance_methods.each { |m| undef_method m unless m =~ /^__/ || m=='instance_eval'}
+        def initialize lexer, context, mappings
+          @lexer, @context, @mappings = lexer, context, mappings
+        end
+
+        def method_missing name
+          name_string = name.to_s          
+          index = name_string =~ /\d+/ ? name_string.slice!(/\d+/).to_i : 0
+          name_string = @mappings[name_string] if @mappings.has_key?(name_string)
+          return @context[name_string][-index] if @context[name_string][-index]
+          Attributes.new
+        end
+      end
+
+      class Attributes
+        instance_methods.each { |m| undef_method m unless m =~ /^__/ }
+        def initialize
+          @attributes = {}
+        end
+
+        def method_missing name, *args
+          name_string = name.to_s
+          return @attributes[name_string] unless name_string =~ /=$/
+          @attributes[name_string.slice(0..-2)] = args.first
+        end
+      end
+    end
+  end
+end

data/lib/aurum/engine/tokenization_facility.rb

@@ -0,0 +1,86 @@
+module Aurum
+  module Engine
+    class Token < Struct.new(:terminal, :value)
+      def to_s
+        terminal
+      end
+    end
+    EOF = Token.new('$eof', '')
+    
+    module BasicTokenizationCapability
+      attr_reader :line, :column
+      def next_symbol
+        return @push_back.pop unless @push_back.empty?
+        @recognized, @recognized_lexeme = nil, ''
+        until @recognized          
+          @lexeme, next_state, char = '', @states.last, nil
+          lookaheads, action = [], nil
+          while next_state
+            if char
+              lookaheads << char
+              @column += 1
+            end
+            return EOF if @input.eof? && @recognized_lexeme.empty? && @lexeme.empty? && lookaheads.empty?
+            state, char = next_state, @input.get_char
+            if lexical_table.actions[state]
+              action = lexical_table.actions[state]
+              while (c = lookaheads.shift)
+                @lexeme << c
+              end
+            end            
+            next_state = goto(state, char)
+          end
+          @input.pushback char
+          unless action
+            unrecognized_lexeme = @recognized_lexeme
+            lookaheads.each {|lookahead| unrecognized_lexeme << lookahead}
+            return Token.new('$unknown', unrecognized_lexeme)
+          end
+          while (c = lookaheads.pop)
+            @input.pushback c   
+          end
+          @recognized_lexeme += @lexeme unless action.token == '$ignored'
+          shift_to(action.state) if action.state
+          @recognized = Token.new(action.token, @recognized_lexeme) if action.token && action.token != '$ignored'
+          instance_eval(&action.action) if action.action 
+        end
+        @recognized
+      end
+      
+      def pushback symbol
+        @push_back.push(symbol)        
+      end
+
+      private
+      def shift_to state
+        @states.push(goto(0, - lexical_table.lexical_states.index(state) - 1))
+      end
+
+      def goto state, input
+        return nil unless input
+        next_state = lexical_table.states[state].find {|tran| tran.character_set.include?(input)}
+        next_state ? next_state.destination : nil
+      end
+    end
+    
+    class PushbackString
+      def initialize string
+        @string = string
+      end
+      
+      def get_char
+        @get_char_index = -1 unless @get_char_index
+        @string[@get_char_index += 1]
+      end
+
+      def eof?
+        @get_char_index = -1 unless @get_char_index
+        @get_char_index >= (@string.length - 1)
+      end
+
+      def pushback char
+        @get_char_index -= 1
+      end
+    end
+  end
+end

data/lib/aurum/grammar.rb

@@ -1,226 +1,59 @@
-module Aurum
-    Symbol.class_eval { attr_accessor :action }
-
-    class << nil
-        attr_accessor :action
-    end
-	
-	Aurum::RecognizeTokenAction.class_eval {attr_accessor :action}
+require 'set'
 
-    class CharacterClassDefinition    	
-        instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
-		attr_reader :definitions
-		
-		def initialize
-			@definitions = {}
-		end
+require File.join(File.dirname(__FILE__), 'grammar/syntax_rules')
+require File.join(File.dirname(__FILE__), 'grammar/lexical_rules')
+require File.join(File.dirname(__FILE__), 'grammar/compiled_tables')
+require File.join(File.dirname(__FILE__), 'grammar/dsl/syntax_definition')
+require File.join(File.dirname(__FILE__), 'grammar/dsl/lexical_definition')
+require File.join(File.dirname(__FILE__), 'grammar/builder/parsing_table_builder')
+require File.join(File.dirname(__FILE__), 'grammar/builder/lexical_table_builder')
 
-        def range a, b
-            a = a[0] if a.kind_of? String
-            b = b[0] if b.kind_of? String
-            CharacterSet::Interval.new(a, b).to_char_set
+module Aurum
+  class Grammar
+    def Grammar.include_grammar(grammar, mapping)
+    end
+    
+    def Grammar.inherited(grammar)
+      grammar.instance_eval do
+        @lexical_rules, @syntax_rules, @semantic_actions = LexicalRules.new, SyntaxRules.new, {}
+        @character_classes_builder = DSL::CharacterClassesBuilder.new
+        @lexical_rules_builder = DSL::LexicalRulesBuilder.new(@lexical_rules, @character_classes_builder)
+        @precedeces_builder = DSL::PrecedencesBuilder.new(@syntax_rules)
+        @productions_builder = DSL::ProductionsBuilder.new(@syntax_rules, @semantic_actions)
+        @reserve_word_states = ['all']
+        
+        def character_classes &definition
+          @character_classes_builder.instance_eval(&definition) if block_given?
         end
-
-        def enum a
-            result = CharacterSet.new
-            result << a
-            result            
+        
+        def tokens &definition
+          @lexical_rules_builder.instance_eval(&definition) if block_given?
+        end
+        
+        def reserve_words_in *states
+          @reserve_word_states = states
+        end
+        
+        def precedences &definition
+          @precedeces_builder.instance_eval(&definition) if block_given?
         end
-		
-        def method_missing name, char_set, &block
-            @definitions[name] = char_set unless @definitions.has_key? name
+        
+        def syntax_rules &definition
+          @productions_builder.instance_eval(&definition) if block_given?
+        end        
+        alias productions syntax_rules
+        
+        
+        def start_from start_symbol
+          start_symbol = Grammar.nonterminal(start_symbol.to_s)
+          @syntax_rules.assign_operator_precedence_to_symbols
+          [Builder.build_parsing_table(@syntax_rules, start_symbol), @semantic_actions]
         end
+        
+        def lexical_table
+          Builder.build_lexical_table(@lexical_rules, @reserve_word_states, @syntax_rules.literals)
+        end
+      end
     end
-
-	class LexicalSpecification		
-		alias __extend extend
-		instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
-		attr_reader :definitions, :character_classes
-			
-		def initialize definition = {:initial => {}}, state = :initial 
-			@definitions, @character_classes = definition, CharacterClassDefinition.new
-			@lexical_definition, @patterns = @definitions[state], {}
-		end
-			
-		def range a, b
-			Pattern.from_char_set CharacterSet::Interval.new(a, b).to_char_set
-		end
-		
-		def string literal
-			Pattern.from_string literal
-		end
-		
-		def enum literal
-			Pattern.from_enum literal
-		end
-				
-		def cat *patterns
-			Pattern.concat *(patterns.collect {|x| x.kind_of?(Pattern) ? x : Pattern.from_string(x.to_s)})
-		end
-					
-		def shift_to state, *patterns, &config
-			pattern = __create_pattern ChangeStateAction.new(state), *patterns
-			within state, &config if block_given?
-			pattern
-		end
-		
-		def match *patterns, &action
-			__create_pattern UserDefinedAction.new(action), *patterns
-		end
-		
-		def ignore *patterns
-			__create_pattern IgnoreAction, *patterns
-		end
-		
-		def within *states, &config
-			for state in states
-				@definitions[state] = {} unless @definitions[state] 
-				LexicalSpecification.new(@definitions, state).instance_eval &config								
-			end
-		end
-		
-		def recognize_and_shift_to token, state, *patterns
-			__create_pattern RecognizeTokenAndChangeStateAction.new(token.to_s, state), *patterns
-		end
-			
-		def method_missing name, *patterns, &action 
-			return __get_pattern(name) if patterns.empty?
-			pattern = Pattern.concat *(patterns.collect {|x| x.kind_of?(Pattern) ? x : Pattern.from_string(x.to_s)}) 			
-			@patterns[name] = pattern
-			if name.to_s =~ /^_/ 
-				recognize_action = RecognizeTokenAction.new(name.to_s)
-				recognize_action.action = action
-				@lexical_definition[pattern] = recognize_action
-			end 
-			pattern
-		end
-		
-		def __get_pattern name
-			return @patterns[name] if @patterns.has_key? name
-			Pattern.from_char_set @character_classes.definitions[name]  
-		end
-					
-		def __create_pattern action, *patterns 
-			pattern = Pattern.concat *(patterns.collect {|x| x.kind_of?(Pattern) ? x : Pattern.from_string(x.to_s)})
-			@lexical_definition[pattern] = action
-			pattern 
-		end			
-	end	
-	
-	class OperatorPrecedenceDefinition
-		instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
-		attr_reader :precedences, :associativities
-		
-		def initialize
-			@precedences, @associativities = [], {:right => [], :left => []}
-		end
-		
-		def operator *operators			
-			operators = operators.collect {|x| __get_symbol x}
-			@precedences << operators.uniq
-		end		
-		
-		def left *operators
-			__associativity :left, *operators
-		end
-		
-		def right *operators
-			__associativity :right, *operators
-		end				
-		
-		def method_missing name
-			Aurum::Symbol.new name.to_s, true
-		end	
-		
-		def __get_symbol operator
-			operator.kind_of?(Aurum::Symbol) ? operator : Aurum::Symbol.new("$literal_#{operator}", true) 
-		end
-		
-		def __associativity direction, *operators
-			symbols = []
-			for operator in operators
-				symbols << (symbol = __get_symbol(operator))
-				@associativities[direction] << symbol unless @associativities[direction].include? symbol				 
-			end			
-			@precedences << symbols.uniq
-		end
-	end
-	
-	class ProductionDefinition
-		instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
-		attr_reader :__definitions, :__literals
-		
-		def initialize
-			@__definitions, @__literals = {}, {}
-		end
-		
-		def method_missing name, *args, &block
-			name_string = name.to_s
-			symbol = case name_string
-				when /^[a-z]/ then Aurum::Symbol.new name_string, false
-				when '_' then nil
-				when /_.+/ then Aurum::Symbol.new name_string, true
-			end
-			symbol.action = block if block_given?
-			(args.empty? || name_string == '_') and return symbol
-			symbols = args.map do |sym|
-				if sym.kind_of? String
-					@__literals[Pattern.from_string(sym)] = RecognizeTokenAction.new "$literal_#{sym}"
-					sym = Aurum::Symbol.new("$literal_#{sym}", true)
-				end 									
-				sym
-			end	
-			@__definitions[symbol] = [].to_set unless @__definitions.has_key? symbol
-			production = Aurum::Production.new symbol, symbols.compact
-			production.action = symbols.last.action if symbols.last.action
-			@__definitions[symbol] << production
-		end			
-	end
-	
-	class Grammar		 			 			
-		def self.character_classes &block
-			@lexical_sepcification = LexicalSpecification.new unless @lexical_sepcification
-			@lexical_sepcification.character_classes.instance_eval &block			
-		end
-		
-		def self.tokens &block
-			@lexical_sepcification = LexicalSpecification.new unless @lexical_sepcification
-			@lexical_sepcification.instance_eval &block
-		end
-		
-		def self.precedences &block
-			@precedences = OperatorPrecedenceDefinition.new unless @precedences
-			@precedences.instance_eval &block
-		end
-		
-		def self.productions &block
-			@productions = ProductionDefinition.new unless @productions
-			@productions.instance_eval &block
-		end
-		
-		def self.lexer input
-			@productions = ProductionDefinition.new unless @productions			
-			@lexical_sepcification.definitions[:initial].merge!(@productions.__literals)
-			generator = Aurum::LexicalTableGenerator.new @lexical_sepcification.definitions
-			table, accepts = generator.lexical_table
-			Aurum::Lexer.new table, accepts, generator.lexical_states, input			
-		end
-		
-		def self.start_from start_symbol
-			@productions = ProductionDefinition.new unless @productions
-			@precedences = OperatorPrecedenceDefinition.new unless @precedences
-			generator = Aurum::ParsingTableGenerator.new @productions.__definitions, @precedences.precedences, @precedences.associativities
-			productions = generator.start_from(Aurum::Symbol.new(start_symbol.to_s, false)).productions
-			table, lookeahead_level = generator.parsing_table
-			Aurum::Parser.new productions, table
-		end
-		
-		def self.method_missing name, input
-			name_string = name.to_s
-			if name_string =~ /^parse_/
-				start_nonterminal = name_string.split(/^parse_/).last
-				self.start_from(start_nonterminal).parse self.lexer(input) 
-			end			
-		end
-	end
+  end  
 end