aurum 0.1.1 → 0.2.0

data/lib/aurum/grammar/automata.rb

@@ -0,0 +1,194 @@
+module Aurum
+  class Grammar
+    class LexicalRules
+      class Automata
+        Transition = Struct.new(:character_set, :destination)
+        attr_reader :table
+        def initialize(table=[])
+          @table = table.is_a?(Array) ? table : Array.new(table){[]}
+        end
+
+        def connect start, character_set, destination
+          @table[start]  << Transition.new(character_set, destination)
+          destination
+        end
+
+        def merge! other
+          start = @table.length
+          for trans in other.table
+            @table << trans.map {|tran| Transition.new(tran.character_set, tran.destination + start)}
+          end
+          start
+        end
+
+        def new_state
+          @table << []
+          @table.size - 1
+        end
+
+        def all_states
+          (0..@table.size - 1).to_a
+        end
+
+        def alphabet states
+          points, reachable_states = [], []
+          for state in states
+            @table[state].each {|tran| points.concat(tran.character_set.to_points(tran.destination))}
+          end
+          points.sort!
+          points.each_with_index do |point, index|
+            if point.is_start
+              reachable_states << point.destination
+            else
+              reachable_states.delete point.destination
+              next if reachable_states.empty?
+            end
+            character_set = range(point, points[index + 1])
+            yield(reachable_states.uniq, character_set) if character_set
+          end
+        end
+
+        def dup
+          Automata.new(@table.map {|x| x.dup})
+        end
+
+        def reverse
+          reverse = Automata.new(@table.size)
+          @table.each_with_index do |state, index|
+            state.each{|tran| reverse.connect(tran.destination, tran.character_set, index)}
+          end
+          reverse
+        end
+        
+        def remove_dead_states accepts
+          dead_states = []
+          @table.each_with_index do |state, index|
+            next if accepts.include?(index) || state.any? {|tran| tran.destination != index }
+            dead_states << index
+          end
+          unless dead_states.empty?
+            @table.each_with_index do |state, index|
+              state.delete_if {|tran| dead_states.include?(tran.destination) }
+            end
+          end
+        end
+
+        private
+        def range point_a, point_b
+          start_point = point_a.is_start ? point_a.char : (point_a.char + 1)
+          end_point = point_b.is_start ? point_b.char - 1 : point_b.char
+          start_point > end_point ? nil : CharacterSet::Interval.new(start_point, end_point).to_char_set
+        end
+      end
+
+      class SubsetDeterminizer
+        def initialize nondeterministic, accepts
+          @unmarked, @dstates, @accepts = [], [], accepts
+          @nondeterministic, @deterministic, @accept_states = nondeterministic, Automata.new, {}
+        end
+
+        def determinize
+          unmark(closure([0]))
+          until @unmarked.empty?
+            start = @unmarked.pop
+            @nondeterministic.alphabet(@dstates[start]) do |states, character_set|
+              destination_state = closure(states)
+              destination = unmark(destination_state) unless destination = @dstates.index(destination_state)
+              @deterministic.connect(start, character_set, destination)
+            end
+          end
+          return @deterministic, @accept_states
+        end
+        private
+        def unmark states
+          @dstates << states
+          @unmarked.push(@deterministic.new_state)
+          accepts = states.find_all {|x| @accepts.include? x}
+          @accept_states[@unmarked.last] = accepts unless accepts.empty?
+          @unmarked.last
+        end
+
+        def closure states
+          closure, unvisited = states.dup, states.dup
+          until unvisited.empty? do
+            @nondeterministic.table[unvisited.pop].each do |tran|
+              if tran.character_set == Epsilon && !closure.include?(tran.destination)
+                closure << tran.destination
+                unvisited << tran.destination
+              end
+            end
+          end
+          closure.sort!
+        end
+      end    
+    
+      class HopcroftMinimizer
+        def initialize deterministic, accepts
+          @deterministic, @accepts = deterministic, accepts
+        end
+
+        def minimize
+          make_initial_partitions
+          refine_partitions
+          automata, accepts =  Aurum::Grammar::LexicalRules::Automata.new(@partitions.size), {}
+          choose_representatives do |representative, index|
+            @deterministic.table[representative].each do |transition|
+              automata.connect(index, transition.character_set, partition_contains(transition.destination))
+            end
+            accepts[index] = @accepts[representative] if @accepts.has_key? representative
+          end
+          automata.remove_dead_states(accepts.keys)
+          return automata, accepts
+        end
+
+        private
+        def make_initial_partitions
+          partitions = {}
+          @accepts.each do |state, action|
+            partitions[action] = [] unless partitions.has_key? action
+            partitions[action] << state
+          end
+          @partitions = [[0], @deterministic.all_states - @accepts.keys - [0]] + partitions.values
+          @partitions.delete []
+        end
+
+        def refine_partitions
+          reverse_automata, working_list = @deterministic.reverse, @partitions.dup
+          until working_list.empty?
+            reverse_automata.alphabet(working_list.pop) do |ia, symbols|
+              @partitions.each do |r|
+                unless r.empty?
+                  r1, r2 = r & ia, r - ia
+                  unless r2.empty? || r2 == r
+                    replace(@partitions, r => [r1, r2])
+                    if working_list.include?(r)
+                      replace(working_list, r => [r1, r2])
+                    else
+                      working_list << (r1.size <= r2.size ? r1 : r2)
+                    end
+                    working_list.uniq!
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def choose_representatives
+          @partitions.each_with_index {|partition, index| yield partition.first, index}
+        end
+
+        def partition_contains state
+          @partitions.each_with_index {|partition, index| return index if partition.include? state}
+        end
+
+        def replace array, replacements
+          replacements.each do |old, new|
+            array.delete old
+            new.each {|x| array << x}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aurum/grammar/builder/augmented_grammar.rb

@@ -0,0 +1,83 @@
+module Aurum
+  module Builder
+    StartSymbol = Aurum::Grammar.nonterminal('$start')
+    
+    def Builder.working_list unvisited
+      yield unvisited, unvisited.pop while !unvisited.empty?
+    end
+    
+    class AugmentedGrammar
+      attr_reader :start_production, :all_productions
+      def initialize syntax_rules, start_symbol
+        @syntax_rules, @start_symbol = syntax_rules, start_symbol
+        @start_production = Aurum::Grammar.production(StartSymbol, [start_symbol])
+        @all_productions = [@start_production]
+        @nonterminals = [StartSymbol, @start_symbol].to_set
+        @nullables = {StartSymbol => false, @start_symbol => false}
+        @first_sets = {StartSymbol => [].to_set, @start_symbol => [].to_set}
+        initialize_for_used_symbols
+      end
+      
+      def nullable? symbol
+        @nullables[symbol]
+      end
+      
+      def first_set symbol
+        @first_sets[symbol]
+      end
+      
+      def productions(nonterminal = nil)
+        return [@start_production] if nonterminal == StartSymbol
+        @syntax_rules.productions(nonterminal)
+      end      
+
+      def compute_nullables
+        fixed_point do |nonterminal|
+          break true if nullable? nonterminal
+          @nullables[nonterminal] = productions(nonterminal).any? {|prod| production_nullable?(prod)}
+        end
+      end
+
+      def compute_first_sets
+        fixed_point do |nonterminal|
+          for production in productions(nonterminal)
+            for symbol in production.symbols
+              is_updated = @first_sets[symbol].inject(false) {|r, s| r |= @first_sets[nonterminal].add?(s) }
+              break unless nullable?(symbol) and !is_updated
+            end
+          end
+          is_updated
+        end
+      end         
+      
+      private
+      def fixed_point
+        changed = true
+        while changed
+          changed = false
+          @nonterminals.each {| nonterminal | changed |= yield(nonterminal)}
+        end
+      end
+      
+      def initialize_for_used_symbols
+        Builder.working_list [@start_symbol] do |unvisited, visiting|
+          for production in @syntax_rules.productions(visiting)
+            @all_productions << production
+            for symbol in production.symbols
+              if symbol.is_terminal
+                @nullables[symbol], @first_sets[symbol] = false, [symbol].to_set unless @nullables.has_key?(symbol)
+              elsif @nonterminals.add?(symbol)
+                unvisited << symbol
+                @nullables[symbol], @first_sets[symbol] = false, [].to_set
+              end              
+            end            
+          end
+        end
+      end
+      
+      def production_nullable? production
+        production.symbols.all? {|symbol| nullable? symbol}
+      end
+    end
+  end
+end

data/lib/aurum/grammar/builder/dot_logger.rb

@@ -0,0 +1,66 @@
+module Aurum
+  module Builder
+    class DotLogger
+      attr_reader :dot_graphs
+      def initialize
+        @dot_graphs = {}
+      end
+
+      def productions productions
+        @productions = productions.to_a
+      end
+
+      def parsing_table_states name, states
+        dot = %Q_digraph grammar {\nrankdir=LR;\n_
+        @productions.each_with_index {|production, index| dot << production_to_dot(production, index)}
+        states.each {|state| dot << state_to_dot(state)}
+        dot << "}\n"
+        @dot_graphs[name] = dot
+      end
+
+      def lexical_table_automata name, automata, accepts
+        dot = %Q_digraph grammar {\nrankdir=LR;\n_
+        accepts.each {|state, action| dot << lexical_action_to_dot(action, state)}
+        automata.table.each_with_index do |transitions, index|
+          dot << %Q{#{index} [shape = circle, color = palegreen, style = filled]\n}
+          dot << %Q{#{index} -> action_#{index}\n} if accepts.has_key?(index)
+          transitions.each {|transition| dot << %Q{#{index} -> #{transition.destination} [label="#{transition.character_set.inspect}"]\n} }
+        end
+        dot << "}\n"
+        @dot_graphs[name] = dot
+      end
+
+      private
+      def state_to_dot state
+        dot = %Q{state_#{state.index} [label = "#{state.empty? ? 'lookahead' : state.all_items.join('\n')}"]\n}
+        dot << %Q{state_#{state.index} [shape = circle, color = #{state.conflict? ? 'maroon' : (state.inconsistent? ? 'yellow' : (state.empty? ? 'lightblue' : 'palegreen'))}, style = filled]\n}
+        for symbol, actions in state.actions
+          for action in actions do
+            dot << %Q{state_#{state.index} -> #{action_to_dot(action)} [label="#{symbol.inspect}"];\n}
+          end
+        end
+        dot
+      end
+
+      def production_to_dot production, index
+        %Q{production_#{index} [label = "#{production.inspect}", color = lightpink, style = filled, shape = octagon];\n}
+      end
+
+      def action_to_dot action
+        if action.is_a?(Aurum::ParsingTable::ShiftAction)
+          "state_#{action.state.index}"
+        else
+          "production_#{@productions.index(action.handle)}"
+        end
+      end
+
+      def lexical_action_to_dot action, state
+        action_attr = []
+        action_attr << "recognize '#{action.token}'" if action.token
+        action_attr << "shift to '#{action.state}'\n" if action.state
+        action_attr << "notify event '#{action.event}'\n" if action.event
+        %Q{action_#{state} [label = "#{action_attr.join("\n")}", color = lightpink, style = filled, shape = octagon];\n}
+      end
+    end
+  end
+end

data/lib/aurum/grammar/builder/lexical_table_builder.rb

@@ -0,0 +1,55 @@
+module Aurum
+  module Builder
+    def Builder.build_lexical_table(lexical_rules, states, literals, logger = nil)
+      LexicalTableBuilder.new(lexical_rules, states, literals).build
+    end
+    
+    class LexicalTableBuilder
+      def initialize lexical_rules, states, literals
+        @patterns, @literals = lexical_rules.patterns, literals
+        @lexical_states = @patterns.keys.to_a - ['all']
+        @literal_states, @literal_patterns, @common_patterns = states, {}, @patterns['all'] || {}
+        for literal in literals
+          pattern = Aurum::Grammar::LexicalRules::Pattern.string(literal.to_s)
+          @literal_patterns[pattern] = Aurum::LexicalTable::Action.new("$literal_#{literal}")          
+        end
+      end
+
+      def build
+        automata = Aurum::Grammar::LexicalRules::Automata.new
+        start, accepts, actions = automata.new_state, {}, {}
+        @lexical_states.each_with_index do |lexical_state, index|
+          lexical_state_start = automata.new_state
+          automata.connect(start, Aurum::Grammar::LexicalRules::CharacterSet.range(-index - 1), lexical_state_start)
+          [@patterns[lexical_state], @common_patterns, literals_in(lexical_state)].each do |patterns|
+            for pattern, action in patterns
+              pattern_start = automata.merge!(pattern.automata)
+              automata.connect(lexical_state_start, Aurum::Grammar::LexicalRules::Epsilon, pattern_start)
+              accepts[pattern_start + pattern.accept] = action
+            end
+          end
+        end
+        automata, dfa_accepts = Aurum::Grammar::LexicalRules::SubsetDeterminizer.new(automata, accepts.keys).determinize
+        for dfa_accept, nfa_accepts in dfa_accepts
+          lexical_actions = nfa_accepts.inject([]) {|result, state| result << accepts[state]}
+          actions[dfa_accept] = resolve(lexical_actions.uniq)
+        end
+        automata, actions = Aurum::Grammar::LexicalRules::HopcroftMinimizer.new(automata, actions).minimize
+        Aurum::LexicalTable.new(automata.table, @lexical_states, actions)
+      end
+
+      private
+      def resolve lexical_actions
+        return lexical_actions.first if lexical_actions.size == 1
+        candidates = lexical_actions.find_all {|action| action.token =~ /^\$literal_/}
+        return candidates.first if candidates.size == 1
+        lexical_actions.each {|action| p action}
+        raise 'conflict'
+      end
+
+      def literals_in state
+        (@literal_states.include?('all') || @literal_states.include?(state)) ? @literal_patterns : {} 
+      end      
+    end
+  end
+end

data/lib/aurum/grammar/builder/parsing_table_builder.rb

@@ -0,0 +1,238 @@
+require File.join(File.dirname(__FILE__), 'augmented_grammar')
+require File.join(File.dirname(__FILE__), 'set_of_items')
+
+module Aurum
+  module Builder
+    def Builder.build_parsing_table(syntax_rules, start_symbol, logger = nil)
+      augmented_grammar = AugmentedGrammar.new(syntax_rules, start_symbol)
+      ParsingTableBuilder.new(augmented_grammar, logger).build
+    end
+    
+    class ParsingTableBuilder
+      def initialize augmented_grammar, logger
+        @augmented_grammar, @logger = augmented_grammar, logger
+        @lookahead_level, @states = 0, []
+        @inconsistent_states = []
+      end
+      
+      def build
+        construct_automata
+        unless @inconsistent_states.empty?
+          @augmented_grammar.compute_nullables
+          @augmented_grammar.compute_first_sets
+          @conflict_states = [].to_set
+          @follow_set_calculator = DigraphTraverser.new do |config|
+            (config.symbol == StartSymbol) ? [false, [Aurum::Grammar::EOF], 65535] : [true, nil, nil]
+          end
+          for inconsistent_state in @inconsistent_states do
+            determine_lookaheads_for(inconsistent_state)
+            @conflict_states << inconsistent_state if inconsistent_state.conflict?
+          end
+          unless @conflict_states.empty?
+            for conflict_state in @conflict_states do
+              for lookahead in conflict_state.conflicts do
+                @state_lookahead_level = 2
+                resolve_conflicts_for(conflict_state, lookahead, Sources.new(conflict_state, lookahead))
+                @lookahead_level = @state_lookahead_level if @state_lookahead_level > @lookahead_level
+              end
+            end
+          end
+        end
+        construct_parsing_table
+      end
+      
+      private
+      def construct_automata
+        item = LRItem.new(@augmented_grammar.start_production)
+        add_state(start_state = State.new(@augmented_grammar, [item]))
+        Builder.working_list [start_state] do |unvisited, visiting|
+          for symbol in visiting.expect_symbols
+            new_state = visiting.goto(symbol)
+            if production = new_state.read_reduce
+              visiting.add_read_reduce(symbol, production)
+            else
+              is_new_state, new_state = add_state(new_state)
+              unvisited << new_state if is_new_state
+              visiting.add_shift(symbol, new_state)
+            end
+          end
+        end
+      end
+      
+      def determine_lookaheads_for state
+        @lookahead_level = 1
+        for item in state.handles
+          lookahead_each(state, item.production) do |production, symbol|
+            if symbol.precedence
+              if production.precedence > symbol.precedence
+                state.actions[symbol].clear
+                state.add_reduce(symbol, production)
+              end              
+            else
+              state.add_reduce(symbol, production)
+            end
+          end
+        end
+      end 
+
+      def resolve_conflicts_for state, lookahead, sources
+        lookahead_state = add_state(State.new(@augmented_grammar, [])).last
+        state.add_lookahead_shift(lookahead, lookahead_state)
+        sources.each_pair do |action, stacks|
+          for stack in stacks
+            raise 'not LALR(n)' if sources.stack_seen.include? stack
+            sources.stack_seen << stack
+            next_lookaheads(stack, lookahead).each {|next_la| lookahead_state.add_action(next_la, action)}
+          end
+        end
+        for symbol in lookahead_state.conflicts
+          @state_lookahead_level += 1
+          resolve_conflicts_for(lookahead_state, symbol, Sources.new(lookahead_state, lookahead, sources, symbol))
+        end if lookahead_state.conflict?
+      end    
+      
+      def construct_parsing_table
+        action_table = @states.map do |state|
+          actions = {}
+          state.actions.each { |symbol, action| actions[symbol.name] = action.first}
+          actions
+        end
+        Aurum::ParsingTable.new(@augmented_grammar.all_productions, action_table, @lookahead_level)
+      end
+      
+      def next_lookaheads stack, lookahead
+        Aurum::Grammar::EOF == lookahead and return [Aurum::Grammar::EOF]
+        top = stack.last
+        lookaheads = top.read_set(lookahead)
+        top.reducable_items(lookahead) do |production, position|
+          c = production.nonterminal
+          if position < stack.length
+            lookaheads |= next_lookaheads stack.slice(0..stack.length-position-1), c
+          else
+            first_part = production.symbols.slice 0..-stack.length-1
+            stack[0].predecessors(first_part).each {|q| lookaheads |= @follow_set_calculator.traverse(Configuration.new(q, c))}
+          end unless c.name == '$start'
+        end
+        lookaheads
+      end      
+      
+      def lookahead_each state, production
+        for predecessor in state.predecessors(production.symbols)
+          config = Configuration.new(predecessor, production.nonterminal)
+          @follow_set_calculator.traverse(config).each {|symbol| yield production, symbol}
+        end
+      end      
+      
+      def add_state state
+        index = @states.index(state)
+        return [false, @states[index]] if index && !state.empty?
+        state.index = @states.size
+        @states << state
+        @inconsistent_states << state if state.inconsistent?
+        [true, state]
+      end       
+    end
+    
+    class Configuration < Struct.new(:state, :symbol)
+      def map_to_set
+        state.read_set(symbol)
+      end
+
+      def result_each
+        state.include_each(symbol) {|state, symbol| yield Configuration.new(state, symbol)}
+      end
+    end
+
+    class DigraphTraverser
+      def initialize &condition
+        @indicitor, @result, @stack = {}, {}, []
+        @condition = condition
+      end
+
+      def traverse node
+        return @result[node] if @indicitor[node]
+        to_be_continued, @result[node], @indicitor[node] = @condition.call(node)
+        if to_be_continued
+          @stack.push(node)
+          @indicitor[node] = (cycle_indicitor = @stack.length)
+          @result[node] = node.map_to_set
+          node.result_each do |new_node|
+            traverse(new_node) unless @indicitor[new_node]
+            @indicitor[node] = [@indicitor[node], @indicitor[new_node]].min
+            @result[node] |= @result[new_node]
+          end
+          node_in_cycle = nil
+          until node_in_cycle == node
+            node_in_cycle = @stack.pop
+            @result[node_in_cycle] = @result[node].dup
+            @indicitor[node_in_cycle] = 65535
+          end if @indicitor[node] == cycle_indicitor
+        end
+        @result[node]
+      end
+    end
+    
+    class Sources < DelegateClass(Hash)
+      attr_reader :stack_seen
+      def initialize(state, symbol, sources = nil, lookahead = nil)
+        @sources, @stack_seen = {}, []
+        for action in state.actions[symbol]
+          @sources[action] = [].to_set
+          if sources
+            @sources[action] = initialize_lookahead_state(sources[action], symbol, lookahead)
+          else
+            @sources[action] = initialize_state(state, action, symbol)
+          end
+        end
+        super(@sources)
+      end
+
+      private
+      def initialize_state state, action, symbol
+        if action.shift_action? || action.read_reduce?
+          stacks = [[state]].to_set
+        else
+          stacks = [].to_set
+          for predecessor in state.predecessors(action.value.symbols)
+            @visited = [].to_set
+            stacks |= follow_sources([predecessor], action.value.nonterminal, symbol)
+          end
+        end
+        stacks
+      end
+      def initialize_lookahead_state stacks, nonterminal, lookahead
+        new_stacks = [].to_set
+        for stack in stacks
+          @visited = [].to_set
+          new_stacks |= follow_sources(stack, nonterminal, lookahead)
+        end
+        new_stacks
+      end
+      def follow_sources(stack, nonterminal, lookahead)
+        top = stack.last
+        if stack.length == 1
+          config = Configuration.new(top, nonterminal)
+          @visited.include?(config) and return []
+          @visited << config
+        end
+        stacks = [].to_set
+        top.read(nonterminal) do |state, direct, indirect|
+          stacks << [state] if direct.include?(lookahead)
+          for indirect_read in indirect
+            stacks |= follow_sources(stack + [state], indirect_read, lookahead)
+          end
+        end
+        top.read_reduce_items(nonterminal) do |production, position|
+          c = production.nonterminal
+          if position < stack.length
+            stacks |= follow_sources stack.slice(0..stack.length-position-1), c, lookahead
+          else
+            first_part = production.symbols.slice 0..-stack.length-1
+            stack[0].predecessors(first_part).each {|q| stacks |= follow_sources([q], c, lookahead)}
+          end unless c == StartSymbol
+        end
+        stacks
+      end
+    end
+  end
+end