aurum 0.1.1 → 0.2.0

data/lib/aurum/grammar/builder/set_of_items.rb

@@ -0,0 +1,190 @@
+require 'delegate'
+
+module Aurum
+  module Builder
+    class LRItem < Struct.new(:production, :position)
+      attr_reader :dot_symbol, :is_kernel, :is_handle, :remaining
+      def initialize(production, position = 0)
+        super production, position
+        @is_handle = position >= production.symbols.size
+        @is_kernel = @is_handle || position != 0 || production.nonterminal == StartSymbol
+        @dot_symbol = production.symbols[position]
+        @remaining = @is_handle ? [] : production.symbols[position..-1]
+      end
+
+      def next
+        LRItem.new(production, position + 1)
+      end
+      
+      def reducable?(symbol)
+        (position == production.symbols.size - 1) && (@dot_symbol == symbol)
+      end
+
+      def inspect
+        result = "#{production.nonterminal.name} -> "
+        production.symbols.each_with_index do |symbol, index|
+          result << '.' if index == position
+          result << symbol.inspect << ' '
+        end
+        result << '.' if position >= production.symbols.length
+        result
+      end
+      alias kernel? is_kernel
+      alias handle? is_handle
+      alias to_s inspect
+    end
+
+    class State < DelegateClass(Array)
+      attr_accessor :index
+      attr_reader :actions, :handles, :all_items, :expect_symbols      
+      def initialize augmented_grammar, items
+        @augmented_grammar = augmented_grammar
+        super(kernels_of_closure(items))
+        @handles = find_all {|item| item.handle?}
+        @actions, @goto_states, @predecessors, @expect_symbols = Hash.new([]), {}, {}, []
+        @all_items.each {|item| @expect_symbols << item.dot_symbol unless item.handle?}        
+      end
+  
+      def inconsistent?
+        @handles.size > 1 || (@handles.size != 0 && @handles.size != size)
+      end
+  
+      def conflict?
+        @actions.values.any? {|actions| actions.size > 1 }
+      end
+  
+      def conflicts
+        conflict = []
+        @actions.each {|symbol, actions| conflict << symbol if actions.size > 1}
+        conflict
+      end
+
+      def read_reduce
+        (@handles.size == 1 && size == 1) ? first.production : nil
+      end      
+  
+      def goto symbol
+        new_state = []
+        @all_items.each {|item| new_state << item.next if item.dot_symbol == symbol}
+        new_state.empty? ? nil : State.new(@augmented_grammar, new_state)
+      end
+  
+      def add_reduce symbol, production
+        add_action(symbol, Aurum::ParsingTable::Action.new(:reduce, production))
+      end
+
+      def add_read_reduce symbol, production
+        add_action(symbol, Aurum::ParsingTable::Action.new(:read_reduce, production))
+      end
+
+      def add_shift symbol, state
+        add_action(symbol, Aurum::ParsingTable::Action.new(:shift, state.index))
+        add_goto_state(symbol, state)
+      end
+      
+      def add_lookahead_shift symbol, state
+        add_goto_state(symbol, state)
+        @actions[symbol] = [Aurum::ParsingTable::Action.new(:lookahead_shift, state.index)]
+      end      
+  
+      def add_action symbol, action
+        @actions[symbol] = [] unless @actions.has_key?(symbol)
+        @actions[symbol] << action unless @actions[symbol].include?(action)
+      end
+  
+      def predecessors(symbols = nil)
+        return @predecessors unless symbols
+        result = [self]
+        symbols.reverse_each do |symbol|
+          new_result = []
+          result.each {|state| new_result |= state.predecessors[symbol] if state.predecessors[symbol]}
+          result.replace(new_result)
+        end
+        result
+      end
+  
+      def read_set symbol
+        return [] unless @expect_symbols.include?(symbol)
+        result, state =[].to_set, @goto_states[symbol]
+        state.each {|item| result |= first_set_of(item.remaining)} if state
+        result
+      end
+  
+      def include_each nonterminal
+        return unless @expect_symbols.include?(nonterminal)
+        for item in @all_items
+          for predecessor in predecessors(item.production.symbols[0, item.position])
+            yield predecessor, item.production.nonterminal
+          end if (item.dot_symbol == nonterminal && nullable?(item.remaining[1..-1]))
+        end
+      end
+  
+      def read nonterminal
+        return unless @expect_symbols.include?(nonterminal)
+        for action in @actions[nonterminal]
+          if action.shift_action?
+            state, direct, indirect = @goto_states[nonterminal], [], []
+            state.expect_symbols.each do |symbol|
+              direct << symbol if symbol.is_terminal
+              indirect << symbol if @augmented_grammar.nullable?(symbol)
+            end
+            yield(state, direct, indirect)
+          end
+        end
+      end
+  
+      def read_reduce_items nonterminal
+        return unless @expect_symbols.include?(nonterminal)
+        for item in @all_items
+          yield(item.production, item.position) if item.reducable?(nonterminal)
+        end
+      end
+
+      def reducable_items nonterminal
+        return unless @expect_symbols.include?(nonterminal)
+        for item in @all_items
+          suffix = item.remaining[1..-1]
+          yield(item.production, item.position) if item.dot_symbol == nonterminal && nullable?(suffix)
+        end
+      end      
+  
+      private
+      def kernels_of_closure items
+        @all_items = items.dup
+        kernel_items = @all_items.find_all {|item| item.kernel?}
+        Builder.working_list items.dup do |unvisited, visiting|
+          symbol = visiting.dot_symbol
+          for production in @augmented_grammar.productions(symbol)
+            item = LRItem.new(production)
+            unless @all_items.include?(item)
+              @all_items << item
+              unvisited << item
+              kernel_items << item if item.kernel?
+            end
+          end unless !symbol || symbol.is_terminal          
+        end
+        kernel_items
+      end
+  
+      def first_set_of symbols
+        result = [].to_set
+        for symbol in symbols
+          result |= @augmented_grammar.first_set(symbol)
+          break unless @augmented_grammar.nullable?(symbol)
+        end
+        result
+      end
+  
+      def nullable? symbols
+        return true if symbols.empty?
+        symbols.all? {|symbol| @augmented_grammar.nullable?(symbol)}
+      end
+      
+      def add_goto_state symbol, state
+        @goto_states[symbol] = state
+        state.predecessors[symbol] = [] unless state.predecessors[symbol]
+        state.predecessors[symbol] << self        
+      end  
+    end
+  end
+end

data/lib/aurum/grammar/compiled_tables.rb

@@ -0,0 +1,20 @@
+module Aurum
+  class ParsingTable < Struct.new(:productions, :actions, :lookahead)
+    class Action < Struct.new(:type, :value)
+      [:shift, :reduce, :lookahead_shift, :read_reduce].each do |type|
+        define_method("#{type}?") { self.type == type }
+      end
+      def shift_action?
+        type == :shift || type == :lookahead_shift
+      end
+      
+      def reduce_action?
+        type == :reduce || type == :read_reduce
+      end
+    end
+  end
+
+  class LexicalTable < Struct.new(:states, :lexical_states, :actions)
+    Action = Struct.new(:token, :state, :event, :action)
+  end
+end

data/lib/aurum/grammar/dsl/lexical_definition.rb

@@ -0,0 +1,94 @@
+module Aurum
+  class Grammar
+    module DSL
+      class CharacterClassesBuilder
+        instance_methods.each { |m| undef_method m unless m =~ /^__/ || m == 'new' || m=='instance_eval'}
+        attr_reader :__named_character_classes__
+
+        def initialize
+          @__named_character_classes__ = {'any' => Aurum::Grammar::LexicalRules::CharacterSet.any,
+            'underscore' => enum('_'),'single_quote' => enum("'"), 'double_quote' => enum('"'),
+            'ascii' => range(0, 255), 'decimal_number' => range(?0, ?9),
+            'ascii_punctuation' => range(33, 47) + range(58, 64) + range(91, 96) + range(123, 126)}
+        end
+
+        def range first, last = first
+          Grammar.range(first, last)
+        end
+
+        def enum literal
+          Grammar.enum(literal)
+        end
+
+        def method_missing name, char_set = nil
+          name = name.to_s
+          return @__named_character_classes__[name] unless char_set
+          raise "already defined character class: #{name}" if @__named_character_classes__.has_key? name
+          @__named_character_classes__[name] = char_set.kind_of?(Aurum::Grammar::LexicalRules::CharacterSet) ? char_set : enum(char_set.to_s)
+        end
+      end
+      
+      class LexicalRulesBuilder
+        instance_methods.each {|m| undef_method m unless m =~ /^__/ || m=='instance_eval'}      
+
+        def initialize lexical_rules, character_classes
+          @lexical_rules, @character_classes = lexical_rules, character_classes
+          @named_patterns, @current_states = {}, [['initial']]
+        end
+
+        def range first, last = first
+          Aurum::Grammar::LexicalRules::Pattern.character_set(Grammar.range(first, last))
+        end
+
+        def enum literal
+          Aurum::Grammar::LexicalRules::Pattern.enum(literal)
+        end
+
+        def string literal
+          Aurum::Grammar::LexicalRules::Pattern.string literal
+        end
+
+        def concat *patterns
+          Aurum::Grammar::LexicalRules::Pattern.concat(patterns.map do
+            |pattern| pattern.is_a?(Aurum::Grammar::LexicalRules::Pattern) ? pattern : string(pattern.to_s)
+          end)
+        end
+
+        def within *states, &definition
+          @saved_named_patterns = @named_patterns.dup
+          @current_states << states.map {|state| state.to_s}
+          instance_eval(&definition)
+          @current_states.pop
+          @named_patterns = @saved_named_patterns
+        end
+
+        def ignore pattern, options = {}, &semantic_action
+          options[:recognize] = '$ignored'
+          match(pattern, options, &semantic_action)
+        end
+
+        def match pattern, options = {}, &semantic_action
+          event = options[:event].to_s if options.has_key?(:event)
+          token = options[:recognize].to_s if options.has_key?(:recognize)
+          lexical_state = options[:shift_to].to_s if options.has_key?(:shift_to)
+          action = Aurum::LexicalTable::Action.new(token, lexical_state, event, semantic_action)
+          pattern = string(pattern.to_s) unless pattern.is_a?(Aurum::Grammar::LexicalRules::Pattern)
+          for state in @current_states.last
+            @lexical_rules.add_lexical_action(state, pattern, action)
+          end
+        end
+
+        def method_missing name, *patterns, &semantic_action
+          name, pattern = name.to_s, concat(*patterns)
+          return match(pattern, :recognize => name, &semantic_action) if name =~ /^_/
+          if patterns.empty?
+            return @named_patterns[name] if @named_patterns.has_key? name
+            return Aurum::Grammar::LexicalRules::Pattern.character_set(@character_classes.__named_character_classes__[name]) if @character_classes.__named_character_classes__.has_key? name
+            raise "can not find pattern named '#{name}'"
+          end
+          @named_patterns[name] = concat(*patterns)
+        end
+      end
+    end
+  end
+end

data/lib/aurum/grammar/dsl/syntax_definition.rb

@@ -0,0 +1,79 @@
+module Aurum
+  class Grammar
+    module DSL
+      class PrecedencesBuilder
+        def initialize syntax_rules
+          @syntax_rules, @level = syntax_rules, 0
+        end
+
+        def nonassoc *operators
+          operator :non_associative, *operators
+        end
+
+        def left *operators
+          operator :left_associative, *operators
+        end
+
+        def right *operators
+          operator :right_associative, *operators
+        end
+
+        def operator associative, *operators
+          for operator in operators
+            name = operator.is_a?(String) ? "$literal_#{operator}" : operator.to_s
+            @syntax_rules.add_operator_precedence name, Aurum::Grammar.precedence(associative, @level)
+          end
+          @level -= 1 unless operators.empty?
+        end
+      end
+      
+      class ProductionsBuilder
+        instance_methods.each {|m| undef_method m unless m =~ /^__/ || m=='instance_eval'}
+        
+        def initialize syntax_rules, actions
+          @syntax_rules, @actions, @symbols = syntax_rules, actions, {}
+        end
+        
+        private
+        def method_missing name, *arguments, &action
+          name = name.to_s
+          __add_symbol__ name
+          symbol = (@symbols[name] || Aurum::Grammar::Epsilon)
+          if arguments.empty?
+            symbol.action = block_given? ? action : nil
+            return symbol
+          end
+          production_name = arguments.pop.to_s if arguments.last.is_a?(Symbol)
+          production_action = action || __action_of_last_grammar_symbol__(arguments)
+          symbols = arguments.inject([]) do |result, arg|
+            break result if arg == Aurum::Grammar::Epsilon
+            result << (arg.is_a?(Aurum::Grammar::SyntaxRules::Symbol) ? arg : __add_literal__(arg))
+          end
+          production = Aurum::Grammar.production(symbol, symbols)
+          production.name = production_name || "$production_#{production.object_id}"
+          result = @syntax_rules.add_syntax_rule(production)
+          @actions[production.name] = production_action if result && production_action
+        end
+        
+        def __add_literal__ literal
+          @syntax_rules.add_literal literal.to_s
+          literal = "$literal_#{literal}"
+          @symbols.has_key?(literal) ? @symbols[literal] : @symbols[literal] = Aurum::Grammar.terminal(literal)
+        end
+
+        def __add_symbol__ name
+          @symbols[name] = case name
+          when /^[a-z]/ : Aurum::Grammar.nonterminal(name)
+          when /^_.+/   : Aurum::Grammar.terminal(name)
+          when /\$literal_/ : Aurum::Grammar.terminal(name)
+          end unless @symbols.has_key?(name) or name == '_'
+        end
+        
+        def __action_of_last_grammar_symbol__ symbols
+          last = symbols.last
+          last.is_a?(Aurum::Grammar::SyntaxRules::Symbol) ? last.action : nil
+        end
+      end
+    end
+  end
+end

data/lib/aurum/grammar/lexical_rules.rb

@@ -0,0 +1,224 @@
+require File.join(File.dirname(__FILE__), 'automata')
+
+module Aurum
+  class Grammar
+    class LexicalRules
+      attr_reader :patterns
+      def initialize
+        @patterns = {}
+      end
+      
+      def add_lexical_action state, pattern, action
+        state_name = state.to_s
+        @patterns[state_name] = {} unless @patterns.has_key?(state_name)
+        @patterns[state_name][pattern] = action
+      end
+      
+      class Pattern
+        def self.string(literal)
+          automata, index = Automata.new(literal.size + 1), 0
+          literal.each_byte {|byte|automata.connect(index, CharacterSet::Interval.new(byte).to_char_set, (index += 1))}
+          Pattern.new(automata, index)
+        end
+
+        def self.enum(literal)
+          character_set(CharacterSet.enum(literal))
+        end
+
+        def self.character_set(char_set)
+          automata = Automata.new(2)
+          Pattern.new(automata, automata.connect(0, char_set, 1))
+        end
+
+        def self.concat patterns
+          automata, index = Automata.new, 0
+          for pattern in patterns
+            index = automata.connect(index, Epsilon, automata.merge!(pattern.automata)) + pattern.accept
+          end
+          Pattern.new(automata, index)
+        end
+
+        attr_reader :automata, :accept
+        def initialize automata, accept
+          @automata, @accept = automata, accept
+        end
+
+        def zero_or_more
+          automata = @automata.dup
+          automata.connect(0, Epsilon, @accept)
+          automata.connect(@accept, Epsilon, 0)
+          Pattern.new(automata, @accept)
+        end
+
+        def one_or_more
+          automata = @automata.dup
+          automata.connect(@accept, Epsilon, 0)
+          Pattern.new(automata, @accept)
+        end
+
+        def zero_or_one
+          automata = @automata.dup
+          automata.connect(0, Epsilon, @accept)
+          Pattern.new(automata, @accept)
+        end
+
+        def not
+          deterministic, accepts = SubsetDeterminizer.new(@automata, [@accept]).determinize
+          sink = deterministic.new_state        
+          deterministic.connect(sink, CharacterSet.any, sink)
+          sink.times do |state|
+            joint = CharacterSet.any
+            deterministic.table[state].each {|tran| joint -= tran.character_set}
+            deterministic.connect(state, joint, sink) unless joint.empty?
+          end
+          accept = deterministic.new_state
+          accept.times {|state| deterministic.connect(state, Epsilon, accept) unless accepts.has_key? state }
+          Pattern.new(deterministic, accept)
+        end
+
+        def [] least, most = least
+          Pattern.concat([self] * least + [self.zero_or_one] * (most-least))
+        end
+
+        def | other
+          automata = Automata.new(2)
+          for pattern in [self, other]
+            automata.connect(automata.connect(0, Epsilon, automata.merge!(pattern.automata)) + pattern.accept, Epsilon, 1)
+          end
+          Pattern.new(automata, 1)
+        end
+
+        def ~
+          any = Pattern.character_set(CharacterSet.any).zero_or_more
+          Pattern.concat([Pattern.concat([any, self, any]).not, self])
+        end
+      end
+      
+      class CharacterSet
+        def self.enum(literal)
+          intervals = []
+          literal.each_byte {|char| intervals << Interval.new(char)}
+          CharacterSet.new(intervals)
+        end
+
+        def self.range(a, b=a)
+          Interval.new(a, b).to_char_set
+        end
+
+        def self.any
+          range(0, 65535)
+        end
+        attr_reader :intervals
+        def initialize(intervals = [])
+          @intervals = intervals
+          merge_intervals
+        end
+
+        def include? char
+          @intervals.any? {|interval| interval.include?(char)}
+        end
+
+        def empty?
+          @intervals.empty?
+        end
+
+        def + other
+          CharacterSet.new(@intervals + other.intervals)
+        end
+
+        def - other
+          intervals = @intervals.dup
+          for interval in other.intervals
+            next unless to_be_replaced = intervals.find {|x| x.include?(interval.first) || x.include?(interval.last)}
+            intervals.delete to_be_replaced
+            intervals << Interval.new(to_be_replaced.first, interval.first-1) if to_be_replaced.first <= interval.first-1
+            intervals << Interval.new(interval.last + 1, to_be_replaced.last) if interval.last + 1 <= to_be_replaced.last
+          end
+          CharacterSet.new(intervals)
+        end
+
+        def to_points destination
+          @intervals.inject [] do |points, interval|
+            points << Point.new(interval.first, true, destination)
+            points << Point.new(interval.last, false, destination)
+          end
+        end
+
+        def == other
+          return false unless other.is_a?(Aurum::Grammar::LexicalRules::CharacterSet)
+          other.intervals == @intervals
+        end
+
+        def inspect
+          @intervals.map{|interval| interval.inspect}.join(',')
+        end
+
+        private
+        def merge_intervals
+          @intervals.sort!
+          merging = nil
+          for interval in @intervals.dup
+            if merging and merging.merge!(interval)
+              @intervals.delete(interval)
+            else
+              merging = interval
+            end
+          end
+        end
+
+        class Interval < Struct.new(:first, :last)
+          include Comparable
+          def initialize first, last = first
+            super first, last
+          end
+          def include? char
+            char = char[0] if char.kind_of? String
+            first <= char && char <= last
+          end
+
+          def merge! other
+            return nil unless include?(other.first) || include?(other.last) || other.first - last == 1 || first - other.last == 1
+            self.first, self.last = [first, other.first].min, [last, other.last].max
+            self
+          end
+
+          def <=> other
+            return first <=> other.first unless first == other.first
+            (last - first) <=> (other.last - other.first)
+          end
+
+          def to_char_set
+            CharacterSet.new([self])
+          end
+
+          def inspect
+            first == last ? chr_of(first) : "#{chr_of(first)}-#{chr_of(last)}"
+          end
+          private
+          def chr_of integer
+            return "##{integer};" if integer < 33
+            integer.chr rescue integer
+          end
+        end
+      end
+      Epsilon = CharacterSet.new
+      
+      class Point < Struct.new(:char, :is_start, :destination)
+        include Comparable
+        def <=> other
+          char == other.char ? (is_start ? (other.is_start ? 0 : -1) : (other.is_start ? 1 : 0)) : (char <=> other.char)
+        end
+      end      
+    end
+    
+    def Grammar.range first, last = first
+      first = first[0] if first.kind_of? String
+      last = last[0] if last.kind_of? String
+      Aurum::Grammar::LexicalRules::CharacterSet.range(first, last)
+    end
+
+    def Grammar.enum literal
+      Aurum::Grammar::LexicalRules::CharacterSet.enum(literal.to_s)
+    end
+  end  
+end