rly 0.1.0 → 0.2.0

data/README.md

@@ -2,8 +2,8 @@
 
 # Rly
 
-Rly is a lexer and parser generator for ruby, based on ideas and solutions of
-Python's [Ply](http://www.dabeaz.com/ply/).
+Rly is a lexer and parser generator for ruby (O RLY?), based on ideas and solutions of
+Python's [Ply](http://www.dabeaz.com/ply/) (in some places it's a total rip off actually).
 
 ## Installation
 

data/assets/ply_dump.erb

@@ -0,0 +1,15 @@
+Created by PLY version 3.4 (http://www.dabeaz.com/ply)
+
+Grammar
+
+<% for p in g.productions %>Rule <%= sprintf("%-5d", p.index) %> <%= p %>
+<% end %>
+Terminals, with rules where they appear
+
+<% @t = g.terminals.keys.map {|k| k.to_s }.sort; for t in @t %><%= sprintf("%-20s : %s", t, (g.terminals[t] || g.terminals[t.to_sym]).join(' ')) %>
+<% end %>
+Nonterminals, with rules where they appear
+
+<% @t = g.nonterminals.keys.sort; for t in @t %><%= sprintf("%-20s : %s", t, g.nonterminals[t].join(' ')) %>
+<% end %>
+<%= backlog %>

data/lib/rly.rb

@@ -1,5 +1,7 @@
 require "rly/version"
 require "rly/lex"
+require "rly/yacc"
+require "rly/parse/rule_parser"
 
 module Rly
   # Your code goes here...

data/lib/rly/lex.rb

@@ -12,10 +12,10 @@ module Rly
   # configuration (check the methods documentation for details).
   #
   # Once you got your lexer configured, you can create its instances passing a
-  # String to be tokenized. You can then use either {#each} method or common
-  # *Enumerable* methods to get the processed tokens.
+  # String to be tokenized. You can then use {#next} method to get tokens. If you
+  # have more string to tokenize, you can append it to input buffer at any time with
+  # {#input}.
   class Lex
-    include Enumerable
 
     # Tracks the current line number for generated tokens
     #
@@ -55,21 +55,46 @@ module Rly
     #   end
     #
     #   lex = MyLexer.new("hello WORLD")
-    #   lex.each do |tok|
-    #     puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
-    #                                        #=> "UPPERS -> WORLD"
-    #   end
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "UPPERS -> WORLD"
+    #   t = lex.next # => nil
     def initialize(input="")
       @input = input
       @pos = 0
       @lineno = 0
     end
 
+    def inspect
+      "#<#{self.class} pos=#{@pos} len=#{@input.length} lineno=#{@lineno}>"
+    end
+
+    # Appends string to input buffer
+    #
+    # The given string is appended to input buffer, further {#next} calls will
+    # tokenize it as usual.
+    #
+    # @api public
+    #
+    # @example
+    #   lex = MyLexer.new("hello")
+    #
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
+    #   t = lex.next # => nil
+    #   lex.input("WORLD")
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "UPPERS -> WORLD"
+    #   t = lex.next # => nil
+    def input(input)
+      @input << input
+    end
+
     # Processes the next token in input
     #
-    # This is the main interface to lexer. If block is given, {#each} behaves like
-    # an usual enumerator, yielding the next token. If there is no block, {#each}
-    # returns an Enumerator object.
+    # This is the main interface to lexer. It returns next available token or **nil**
+    # if there are no more tokens available in the input string.
     #
     # {#each} Raises {LexError} if the input cannot be processed. This happens if
     # there were no matches by 'token' rules and no matches by 'literals' rule.
@@ -78,23 +103,19 @@ module Rly
     # after returning from error handler is still unchanged.
     #
     # @api public
-    # @yieldparam tok [LexToken] next processed token
     # @raise [LexError] if the input cannot be processed
-    # @return [Enumerator] if block is not given
-    # @return [nil] if block is given
+    # @return [LexToken] if the next chunk of input was processed successfully
+    # @return [nil] if there are no more tokens available in input
     #
     # @example
     #   lex = MyLexer.new("hello WORLD")
     #
-    #   lex.each #=> #<Enumerator: ...>
-    #
-    #   lex.each do |tok|
-    #     puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
-    #                                        #=> "UPPERS -> WORLD"
-    #   end
-    def each
-      return self.to_enum unless block_given?
-
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
+    #   t = lex.next
+    #   puts "#{tok.type} -> #{tok.value}" #=> "UPPERS -> WORLD"
+    #   t = lex.next # => nil
+    def next
       while @pos < @input.length
         if self.class.ignores_list[@input[@pos]]
           @pos += 1
@@ -105,15 +126,17 @@ module Rly
         self.class.tokens.each do |type, rule, block|
           m = rule.match(@input, @pos)
           next unless m
+          next unless m.begin(0) == @pos
 
           tok = LexToken.new(type, m[0], self)
 
           matched = true
 
           tok = block.call(tok) if block
-          yield tok if tok.type
 
           @pos = m.end(0)
+
+          return tok if tok.type
         end
 
         unless matched
@@ -121,8 +144,10 @@ module Rly
             tok = LexToken.new(@input[@pos], @input[@pos], self)
 
             matched = true
-            yield tok
+
             @pos += 1
+
+            return tok
           end
         end
 
@@ -134,16 +159,20 @@ module Rly
             if pos == @pos
               raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
             else
-              yield tok if tok && tok.type
+              return tok if tok && tok.type
             end
           else
             raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
           end
         end
       end
+      return nil
     end
 
     class << self
+      def terminals
+        self.tokens.map { |t,r,b| t }.compact + self.literals_list.chars.to_a
+      end
       # Returns the list of registered tokens
       #
       # @api private

data/lib/rly/lex_token.rb

@@ -9,5 +9,13 @@ module Rly
       @value = value
       @lexer = lexer
     end
+
+    def to_s
+      @value.to_s
+    end
+
+    def inspect
+      "#<LexToken #{@type} '#{@value}'>"
+    end
   end
 end

data/lib/rly/parse/grammar.rb

@@ -0,0 +1,211 @@
+require "rly/parse/production"
+require "rly/parse/lr_item"
+
+module Rly
+  class Grammar
+    attr_reader :terminals, :nonterminals, :productions, :prodnames, :start, :precedence
+
+    def initialize(terminals)
+      @productions = [nil]
+      @prodnames = {}
+      @prodmap = {}
+
+      @terminals = {}
+      terminals.each do |t|
+        raise ArgumentError unless t.upcase == t
+        @terminals[t] = []
+      end
+      @terminals[:error] = []
+
+      @nonterminals = {}
+      @first = {}
+      @follow = {}
+      @precedence = {}
+      @used_precedence = {}
+      @start = nil
+    end
+
+    def add_production(name, symbols, &block)
+      raise ArgumentError unless name.downcase == name
+      raise ArgumentError if name == :error
+
+      symbols.each do |sym|
+        if sym.is_a?(String)
+          raise ArgumentError unless sym.length == 1
+          @terminals[sym] = [] unless @terminals[sym]
+        end
+      end
+
+      precedence = prec_for_rightmost_terminal(symbols)
+
+      mapname = "#{name.to_s} -> #{symbols.to_s}"
+      raise ArgumentError if @prodmap[mapname]
+
+      index = @productions.count
+      @nonterminals[name] = [] unless @nonterminals[name]
+
+      symbols.each do |sym|
+        if @terminals[sym]
+          @terminals[sym] << index
+        else
+          @nonterminals[sym] = [] unless @nonterminals[sym]
+          @nonterminals[sym] << index
+        end
+      end
+
+      p = Production.new(index, name, symbols, precedence, block)
+
+      @productions << p
+      @prodmap[mapname] = p
+
+      @prodnames[name] = [] unless @prodnames[name]
+      @prodnames[name] << p
+
+      p
+    end
+
+    def set_precedence(term, assoc, level)
+      raise RuntimeError if @productions != [nil]
+      raise ArgumentError if @precedence[term]
+      raise ArgumentError unless [:left, :right, :noassoc].include?(assoc)
+
+      @precedence[term] = [assoc, level]
+    end
+
+    def set_start(symbol=nil)
+      symbol = @productions[1].name unless symbol
+      raise ArgumentError unless @nonterminals[symbol]
+      @productions[0] = Production.new(0, :"S'", [symbol])
+      @nonterminals[symbol] << 0
+      @start = symbol
+    end
+
+    def build_lritems
+      @productions.each do |p|
+        lastlri = p
+        i = 0
+        lr_items = []
+        while true do
+          if i > p.length
+            lri = nil
+          else
+            lri = LRItem.new(p,i)
+            lri.lr_after = @prodnames[lri.prod[i+1]] || []
+            lri.lr_before = lri.prod[i-1] || nil
+          end
+
+          lastlri.lr_next = lri
+          break unless lri
+          lr_items << lri
+          lastlri = lri
+          i += 1
+        end
+        p.lr_items = lr_items
+      end
+    end
+
+    def compute_first
+      return @first unless @first.empty?
+
+      @terminals.keys.each { |t| @first[t] = [t] }
+      @first[:'$end'] = [:'$end']
+      @nonterminals.keys.each { |n| @first[n] = [] }
+      while true
+        any_changes = false
+        nonterminals.keys.each do |n|
+          @prodnames[n].each do |p|
+            _first(p.prod).each do |f|
+              unless @first[n].include?(f)
+                @first[n] << f
+                any_changes = true
+              end
+            end
+          end
+        end
+        break unless any_changes
+      end
+
+      @first
+    end
+
+    def compute_follow(start=nil)
+      return @follow unless @follow.empty?
+
+      compute_first if @first.empty?
+
+      @nonterminals.keys.each { |n| @follow[n] = [] }
+
+      start = @productions[1].name unless start
+
+      @follow[start] = [:'$end']
+
+      while true
+        didadd = false
+        @productions[1..-1].each do |p|
+          p.prod.length.times do |i|
+            b = p.prod[i]
+            next unless @nonterminals.include?(b)
+
+            fst = _first(p.prod[i+1..-1])
+            hasempty = false
+            fst.each do |f|
+              if f != :'<empty>' && !@follow[b].include?(f)
+                @follow[b] << f
+                didadd = true
+              end
+              hasempty = true if f == :'<empty>'
+            end
+            if hasempty || i == p.prod.length - 1
+              @follow[p.name].each do |f|
+                unless @follow[b].include?(f)
+                  @follow[b] << f
+                  didadd = true
+                end
+              end
+            end
+          end
+        end
+        break unless didadd
+      end
+
+      @follow
+    end
+
+    private
+    def _first(beta)
+      result = []
+      should_add_empty = true
+
+      beta.each do |x|
+        x_produces_empty = false
+
+        @first[x].each do |f|
+          if f == :'<empty>'
+            x_produces_empty = true
+          else
+            result << f unless result.include?(f)
+          end
+        end
+
+        if x_produces_empty
+          next
+        else
+          should_add_empty = false
+          break
+        end
+      end
+      result << :'<empty>' if should_add_empty
+
+      result
+    end
+
+    def prec_for_rightmost_terminal(symbols)
+      symbols.reverse_each do |sym|
+        next unless @terminals[sym]
+
+        return @precedence[sym] || [:right, 0]
+      end
+      [:right, 0]
+    end
+  end
+end

data/lib/rly/parse/lr_item.rb

@@ -0,0 +1,32 @@
+module Rly
+  class LRItem
+    attr_accessor :lr_after, :lr_before, :lr_next
+    attr_reader :prod, :name, :usyms, :lr_index, :length, :lookaheads, :index
+
+    def initialize(p, n)
+      @name = p.name
+      @prod = p.prod.dup
+      @index = p.index
+      @lr_index = n
+      @lookaheads = {}
+      @prod.insert(n, :'.')
+      @length = @prod.length
+      @usyms = p.usyms
+
+      @lr_items = []
+      @lr_next = nil
+    end
+    
+    def to_s
+      if @prod
+        "#{@name} -> #{@prod.join(' ')}"
+      else
+        "#{@name} -> <empty>"
+      end
+    end
+
+    def inspect
+      "#<LRItem #{to_s}>"
+    end
+  end
+end

data/lib/rly/parse/lr_table.rb

@@ -0,0 +1,529 @@
+require "set"
+require "rly/parse/ply_dump"
+
+module Rly
+  class LRTable
+    MAXINT = (2**(0.size * 8 -2) -1)
+
+    attr_reader :lr_action, :lr_goto, :lr_productions
+
+    def initialize(grammar, method=:LALR)
+      raise ArgumentError unless [:LALR, :SLR].include?(method)
+
+      @grammar = grammar
+      @lr_method = method
+
+      @lr_action = {}
+      @lr_goto = {}
+      @lr_productions = grammar.productions
+      @lr_goto_cache = {}
+      @lr0_cidhash = {}
+
+      @add_count = 0
+
+      @sr_conflict = 0
+      @rr_conflict = 0
+      @conflicts = []
+
+      @sr_conflicts = []
+      @rr_conflicts = []
+
+      grammar.build_lritems
+      grammar.compute_first
+      grammar.compute_follow
+    end
+
+    def parse_table(log=PlyDump.stub)
+      productions = @grammar.productions
+      precedence = @grammar.precedence
+
+      actionp = {}
+
+      log.info("Parsing method: %s", @lr_method)
+
+      c = lr0_items
+
+      add_lalr_lookaheads(c) if @lr_method == :LALR
+
+      # Build the parser table, state by state
+      st = 0
+      c.each do |i|
+        # Loop over each production in I
+        actlist = []              # List of actions
+        st_action  = {}
+        st_actionp = {}
+        st_goto    = {}
+        log.info("")
+        log.info("state %d", st)
+        log.info("")
+        i.each { |p| log.info("    (%d) %s", p.index, p.to_s) }
+        log.info("")
+
+        i.each do |p|
+          if p.length == p.lr_index + 1
+            if p.name == :"S'"
+              # Start symbol. Accept!
+              st_action[:"$end"] = 0
+              st_actionp[:"$end"] = p
+            else
+              # We are at the end of a production.  Reduce!
+              if @lr_method == :LALR
+                laheads = p.lookaheads[st]
+              else
+                laheads = @grammar.follow[p.name]
+              end
+              laheads.each do |a|
+                actlist << [a, p, sprintf("reduce using rule %d (%s)", p.index, p)]
+                r = st_action[a]
+                if r
+                  # Whoa. Have a shift/reduce or reduce/reduce conflict
+                  if r > 0
+                    # Need to decide on shift or reduce here
+                    # By default we favor shifting. Need to add
+                    # some precedence rules here.
+                    sprec, slevel = productions[st_actionp[a].number].prec
+                    rprec, rlevel = precedence[a] || [:right, 0]
+                    if (slevel < rlevel) || ((slevel == rlevel) && (rprec == :left))
+                      # We really need to reduce here.
+                      st_action[a] = -p.number
+                      st_actionp[a] = p
+                      if ! slevel && ! rlevel
+                        log.info("  ! shift/reduce conflict for %s resolved as reduce",a)
+                        @sr_conflicts << [st, a, 'reduce']
+                      end
+                      productions[p.number].reduced += 1
+                    elsif (slevel == rlevel) && (rprec == :nonassoc)
+                      st_action[a] = nil
+                    else
+                      # Hmmm. Guess we'll keep the shift
+                      unless rlevel
+                        log.info("  ! shift/reduce conflict for %s resolved as shift",a)
+                        @sr_conflicts << [st,a,'shift']
+                      end
+                    end
+                  elsif r < 0
+                      # Reduce/reduce conflict.   In this case, we favor the rule
+                      # that was defined first in the grammar file
+                      oldp = productions[-r]
+                      pp = productions[p.number]
+                      if oldp.line > pp.line
+                        st_action[a] = -p.number
+                        st_actionp[a] = p
+                        chosenp = pp
+                        rejectp = oldp
+                        productions[p.number].reduced += 1
+                        productions[oldp.number].reduced -= 1
+                      else
+                        chosenp,rejectp = oldp,pp
+                      end
+                      @rr_conflicts << [st, chosenp, rejectp]
+                      log.info("  ! reduce/reduce conflict for %s resolved using rule %d (%s)", a, st_actionp[a].number, st_actionp[a])
+                  else
+                    raise RuntimeError("Unknown conflict in state #{st}")
+                  end
+                else
+                  st_action[a] = -p.index
+                  st_actionp[a] = p
+                  productions[p.index].reduced += 1
+                end
+              end
+            end
+          else # <-- level ok
+            # i = p.lr_index
+            a = p.prod[p.lr_index+1]       # Get symbol right after the "."
+            if @grammar.terminals.include?(a)
+              g = lr0_goto(i, a)
+              j = @lr0_cidhash[g.hash] || -1
+              if j >= 0
+                # We are in a shift state
+                actlist << [a, p, sprintf("shift and go to state %d", j)]
+                r = st_action[a]
+                if r
+                  # Whoa have a shift/reduce or shift/shift conflict
+                  if r > 0
+                    if r != j
+                      raise RuntimeError("Shift/shift conflict in state #{st}")
+                    end
+                  elsif r < 0
+                    # Do a precedence check.
+                    #   -  if precedence of reduce rule is higher, we reduce.
+                    #   -  if precedence of reduce is same and left assoc, we reduce.
+                    #   -  otherwise we shift
+                    rprec, rlevel = productions[st_actionp[a].index].precedence
+                    sprec, slevel = precedence[a] || [:right, 0]
+                    if (slevel > rlevel) || ((slevel == rlevel) && (rprec == :right))
+                      # We decide to shift here... highest precedence to shift
+                      productions[st_actionp[a].index].reduced -= 1
+                      st_action[a] = j
+                      st_actionp[a] = p
+                      unless rlevel
+                        log.info("  ! shift/reduce conflict for %s resolved as shift",a)
+                        @sr_conflicts << [st, a, 'shift']
+                      end
+                    elsif (slevel == rlevel) && (rprec == :nonassoc)
+                      st_action[a] = nil
+                    else
+                      # Hmmm. Guess we'll keep the reduce
+                      if ! slevel && ! rlevel
+                        log.info("  ! shift/reduce conflict for %s resolved as reduce",a)
+                        @sr_conflicts << [st, a, 'reduce']
+                      end
+                    end
+                  else
+                    raise RuntimeError("Unknown conflict in state #{st}")
+                  end
+                else
+                  st_action[a] = j
+                  st_actionp[a] = p
+                end
+              end
+            end
+          end
+        end
+
+        # Print the actions associated with each terminal
+        _actprint = {}
+        actlist.each do |a, p, m|
+          if st_action[a]
+            if p == st_actionp[a]
+              log.info("    %-15s %s",a,m)
+              _actprint[[a,m]] = 1
+            end
+          end
+        end
+        log.info("")
+        # Print the actions that were not used. (debugging)
+        not_used = false
+        actlist.each do |a, p, m|
+          if st_action[a]
+            unless p == st_actionp[a]
+              unless _actprint[[a,m]]
+                log.debug("  ! %-15s [ %s ]", a, m)
+                not_used = true
+                _actprint[[a,m]] = 1
+              end
+            end
+          end
+        end
+        log.debug("") if not_used
+
+        # Construct the goto table for this state
+
+        nkeys = {}
+        i.each do |ii|
+          ii.usyms.each do |s|
+            nkeys[s] = nil if @grammar.nonterminals.include?(s)
+          end
+        end
+        nkeys.each do |n, _|
+          g = lr0_goto(i, n)
+          j = @lr0_cidhash[g.hash] || -1
+          if j >= 0
+            st_goto[n] = j
+            log.info("    %-30s shift and go to state %d",n,j)
+          end
+        end
+
+        @lr_action[st] = st_action
+        actionp[st] = st_actionp
+        @lr_goto[st] = st_goto
+        st += 1
+      end
+    end
+
+    private
+    def add_lalr_lookaheads(c)
+      nullable = compute_nullable_nonterminals
+      trans = find_nonterminal_transitions(c)
+      readsets = compute_read_sets(c, trans, nullable)
+      lookd, included = compute_lookback_includes(c, trans, nullable)
+      followsets = compute_follow_sets(trans, readsets, included)
+      add_lookaheads(lookd, followsets)
+    end
+
+    def lr0_closure(i)
+      @add_count += 1
+
+      # Add everything in I to J
+      j = i.dup
+      didadd = true
+      while didadd
+        didadd = false
+        j.each do |k|
+          k.lr_after.each do |x|
+            next if x.lr0_added == @add_count
+            # Add B --> .G to J
+            j << x.lr_next
+            x.lr0_added = @add_count
+            didadd = true
+          end
+        end
+      end
+      j
+    end
+
+    def lr0_goto(i, x)
+      g = @lr_goto_cache[[i.hash, x]]
+      return g if g
+
+      s = @lr_goto_cache[x]
+      unless s
+        s = {}
+        @lr_goto_cache[x] = s
+      end
+
+      gs = []
+      i.each do |p|
+        n = p.lr_next
+        if n and n.lr_before == x
+          s1 = s[n.hash]
+          unless s1
+            s1 = {}
+            s[n.hash] = s1
+          end
+          gs << n
+          s = s1
+        end
+      end
+      g = s[:'$end']
+      unless g
+        if gs
+          g = lr0_closure(gs)
+          s[:'$end'] = g
+        else
+          s[:'$end'] = gs
+        end
+      end
+      @lr_goto_cache[[i.hash,x]] = g
+      g
+    end
+
+    def lr0_items
+      c = [ lr0_closure([@grammar.productions[0].lr_next]) ]
+
+      c.each_with_index { |c_i, j| @lr0_cidhash[c_i.hash] = j }
+
+      i = 0
+      while i < c.length
+        c_i = c[i]
+        i += 1
+
+        asyms = Set.new
+        c_i.each { |ii| ii.usyms.each { |s| asyms << s } }
+
+        asyms.each do |x|
+          g = lr0_goto(c_i, x)
+          next if g.empty?
+          next if @lr0_cidhash[g.hash]
+          @lr0_cidhash[g.hash] = c.length
+          c << g
+        end
+      end
+      c
+    end
+
+    def compute_nullable_nonterminals
+      nullable = {}
+      num_nullable = 0
+      while true
+        @grammar.productions[1..-1].each do |p|
+          if p.length == 0
+            nullable[p.name] = 1
+            next
+          end
+          found_t = false
+          p.prod.each do |t|
+            unless nullable[t]
+              found_t = true
+              break
+            end
+          end
+          nullable[p.name] = 1 unless found_t
+        end
+        break if nullable.length == num_nullable
+        num_nullable = nullable.length
+      end
+      nullable
+    end
+
+    def find_nonterminal_transitions(c)
+      trans = []
+      c.each_with_index do |a, state|
+        a.each do |p|
+          if p.lr_index < p.length - 1
+            next_prod = p.prod[p.lr_index+1]
+            if @grammar.nonterminals[next_prod]
+              t = [state, next_prod]
+              trans << t unless trans.include?(t)
+            end
+          end
+        end
+      end
+      trans
+    end
+
+    def compute_read_sets(c, ntrans, nullable)
+      fp = lambda { |x| dr_relation(c, x, nullable) }
+      r = lambda { |x| reads_relation(c, x, nullable) }
+      digraph(ntrans, r, fp)
+    end
+
+    def dr_relation(c, trans, nullable)
+      dr_set = {}
+      state, n = trans
+      terms = []
+
+      g = lr0_goto(c[state], n)
+      g.each do |p|
+        if p.lr_index < p.length - 1
+          a = p.prod[p.lr_index+1]
+          if @grammar.terminals.include?(a)
+            terms << a unless terms.include?(a)
+          end
+        end
+      end
+
+      terms << :'$end' if state == 0 && n == @grammar.productions[0].prod[0]
+      
+      terms
+    end
+
+    def reads_relation(c, trans, empty)
+        rel = []
+        state, n = trans
+
+        g = lr0_goto(c[state], n)
+        j = @lr0_cidhash[g.hash] || -1
+        g.each do |p|
+          if p.lr_index < p.length - 1
+            a = p.prod[p.lr_index + 1]
+            rel << [j, a] if empty.include?(a)
+          end
+        end
+
+        rel
+    end
+
+    def digraph(x, r, fp)
+      n = {}
+      x.each { |xx| n[xx] = 0 }
+      stack = []
+      f = {}
+      x.each do |xx|
+        traverse(xx, n, stack, f, x, r, fp) if n[xx] == 0
+      end
+      f
+    end
+
+    def traverse(xx, n, stack, f, x, r, fp)
+      stack.push(xx)
+      d = stack.length
+      n[xx] = d
+      f[xx] = fp.call(xx)
+
+      rel = r.call(xx)
+      rel.each do |y|
+        traverse(y, n, stack, f, x, r, fp) if n[y] == 0
+        
+        n[xx] = [n[xx], n[y]].min
+
+        arr = f[y] || []
+        arr.each do |a|
+          f[xx] << a unless f[xx].include?(a)
+        end
+      end
+      if n[xx] == d
+        n[stack[-1]] = MAXINT
+        f[stack[-1]] = f[xx]
+        element = stack.pop()
+        while element != xx
+          n[stack[-1]] = MAXINT
+          f[stack[-1]] = f[xx]
+          element = stack.pop()
+        end
+      end
+    end
+
+    def compute_lookback_includes(c, trans, nullable)
+      lookdict = {}
+      includedict = {}
+
+      dtrans = trans.each_with_object({}) { |k, h| h[k] = 1 }
+
+      trans.each do |state, n|
+        lookb = []
+        includes = []
+        c[state].each do |p|
+          next unless p.name == n
+
+          lr_index = p.lr_index
+          j = state
+          while lr_index < p.length - 1
+            lr_index = lr_index + 1
+            t = p.prod[lr_index]
+
+            if dtrans.include?([j,t])
+              li = lr_index + 1
+              escaped = false
+              while li < p.length
+                if @grammar.terminals[p.prod[li]]
+                  escaped = true
+                  break
+                end
+                unless nullable[p.prod[li]]
+                  escaped = true
+                  break
+                end
+                li = li + 1
+              end
+              includes << [j,t] unless escaped
+            end
+
+            g = lr0_goto(c[j],t)
+            j = @lr0_cidhash[g.hash] || -1
+          end
+
+          c[j].each do |r|
+            next unless r.name == p.name
+            next unless r.length == p.length
+            i = 0
+            escaped = false
+            while i < r.lr_index
+              unless r.prod[i] == p.prod[i+1]
+                escaped = true
+                break
+              end
+              i = i + 1
+            end
+            lookb << [j,r] unless escaped
+          end
+        end
+        includes.each do |i|
+          includedict[i] = [] unless includedict[i]
+          includedict[i] << [state, n]
+        end
+        lookdict[[state,n]] = lookb
+      end
+
+      [lookdict, includedict]
+    end
+
+    def compute_follow_sets(ntrans, readsets, inclsets)
+      fp = lambda { |x| readsets[x] }
+      r  = lambda { |x| inclsets[x] || [] }
+      digraph(ntrans, r, fp)
+    end
+
+    def add_lookaheads(lookbacks, followset)
+      lookbacks.each do |trans, lb|
+        lb.each do |state, p|
+          p.lookaheads[state] = [] unless p.lookaheads[state]
+          f = followset[trans] || []
+          f.each do |a|
+            p.lookaheads[state] << a unless p.lookaheads[state].include?(a)
+          end
+        end
+      end
+    end
+  end
+end