rly 0.1.0 → 0.2.0

data/lib/rly/parse/ply_dump.rb

@@ -0,0 +1,52 @@
+require "rly/parse/lr_table"
+
+module Rly
+
+  class PlyDump
+    attr_reader :backlog
+
+    def initialize(grammar)
+      @grammar = grammar
+      @backlog = ""
+      if grammar
+        @t = Rly::LRTable.new(grammar)
+        @t.parse_table(self)
+      end
+    end
+
+    def to_s
+      fn = File.join(File.dirname(__FILE__), '..', '..', '..', 'assets', 'ply_dump.erb')
+      e = ERB.new(open(fn).read)
+      e.result(TinyContext.new(g: @grammar, backlog: @backlog).get_binding)
+    end
+
+    def self.stub
+      PlyDump.new(nil)
+    end
+
+    def info(*args)
+      s = sprintf(*args)
+      @backlog += s + "\n"
+    end
+
+    def debug(*args)
+      s = sprintf(*args)
+      @backlog += s + "\n"
+    end
+
+    class TinyContext
+      def initialize(ctx)
+        @ctx = ctx
+      end
+
+      def get_binding
+        binding()
+      end
+
+      def method_missing(m)
+        @ctx[m]
+      end
+    end
+  end
+
+end

data/lib/rly/parse/production.rb

@@ -0,0 +1,38 @@
+module Rly
+  class Production
+    attr_reader :index, :name, :prod, :precedence, :block, :usyms, :line
+    attr_accessor :lr_items, :lr_next, :lr0_added, :reduced
+
+    # FIXME line!!!
+    def initialize(index, name, prod, precedence=[:right, 0], block=nil)
+      @index = index
+      @name = name
+      @prod = prod
+      @precedence = precedence
+      @block = block
+
+      @usyms = []
+      prod.each { |sym| @usyms << sym unless @usyms.include?(sym) }
+    end
+
+    def to_s
+      "#{name} -> #{@prod.map { |s| s.to_s }.join(' ')}"
+    end
+
+    def inspect
+      "#<Production #{to_s}>"
+    end
+
+    def length
+      @prod.length
+    end
+
+    def lr0_added
+      @lr0_added ||= 0
+    end
+
+    def reduced
+      @reduced ||= 0
+    end
+  end
+end

data/lib/rly/parse/rule_parser.rb

@@ -0,0 +1,68 @@
+require "rly/lex"
+require "rly/parse/grammar"
+require "rly/parse/lr_table"
+
+module Rly
+  class RuleParser < Yacc
+    def self.lexer_class
+      return @lexer_class if @lexer_class
+
+      @lexer_class = Class.new(Lex) do
+        token :ID, /[a-zA-Z_][a-zA-Z_0-9]*/
+        token :LITERAL, /"."|'.'/ do |t|
+          t.value = t.value[1]
+          t
+        end
+        literals ":|"
+        ignore " \t\n"
+      end
+
+      @lexer_class
+    end
+
+    def grammar
+      return @grammar if @grammar
+
+      @grammar = Grammar.new(self.class.lexer_class.terminals)
+
+      @grammar.add_production(:grammar, [:ID, ':', :rules]) do |g, pname, _, r|
+        productions = []
+        r.value.each do |p|
+          productions << [pname.value.to_sym, p]
+        end
+        g.value = productions
+      end
+      @grammar.add_production(:rules, [:rule, '|', :rules]) do |rls, r, _, rl|
+        rls.value = [r.value] + rl.value
+      end
+      @grammar.add_production(:rules, [:rule]) do |rl, r|
+        rl.value = [r.value]
+      end
+      @grammar.add_production(:rule, [:tokens]) do |r, tok|
+        r.value = tok.value
+      end
+      @grammar.add_production(:tokens, [:ID, :tokens]) do |t, i, toks|
+        t.value = [i.value.to_sym] + toks.value
+      end
+      @grammar.add_production(:tokens, [:LITERAL, :tokens]) do |t, l, toks|
+        t.value = [l.value] + toks.value
+      end
+      @grammar.add_production(:tokens, [:ID]) do |t, i|
+        t.value = [i.value.to_sym]
+      end
+      @grammar.add_production(:tokens, [:LITERAL]) do |t, l|
+        t.value = [l.value]
+      end
+
+      @grammar.set_start
+
+      @grammar.build_lritems
+
+      @lr_table = LRTable.new(@grammar)
+
+      @lr_table.parse_table
+
+      @grammar
+    end
+  end
+end

data/lib/rly/parse/yacc_production.rb

@@ -0,0 +1,11 @@
+module Rly
+  class YaccProduction
+    attr_accessor :lexer, :parser, :stack, :slice
+
+    def initialize(slice, stack=nil)
+      @slice = slice
+      @stack = stack
+    end
+    
+  end
+end

data/lib/rly/parse/yacc_symbol.rb

@@ -0,0 +1,6 @@
+module Rly
+  class YaccSymbol
+    attr_accessor :type, :value, :lineno, :endlineno, :lexpos, :endlexpos
+
+  end
+end

data/lib/rly/version.rb

@@ -1,3 +1,4 @@
 module Rly
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
+  PLY_BASE_VERSION = "3.4"
 end

data/lib/rly/yacc.rb

@@ -0,0 +1,355 @@
+require "rly/lex"
+require "rly/parse/grammar"
+require "rly/parse/yacc_production"
+require "rly/parse/yacc_symbol"
+
+module Rly
+  class YaccError < RuntimeError; end
+
+  class Yacc
+    attr_reader :lex, :grammar, :lr_table
+
+    def initialize(lex=nil)
+      raise ArgumentError.new("No lexer available") if lex == nil && self.class.lexer_class == nil
+      @lex = lex || self.class.lexer_class.new
+
+      @grammar = grammar
+    end
+
+    def parse(input=nil)
+      lookahead = nil
+      lookaheadstack = []
+      actions = @lr_table.lr_action
+      goto = @lr_table.lr_goto
+      prod = @lr_table.lr_productions
+      pslice = YaccProduction.new(nil)
+      errorcount = 0
+
+      # Set up the lexer and parser objects on pslice
+      pslice.lexer = @lex
+      pslice.parser = self
+
+      # If input was supplied, pass to lexer
+      @lex.input(input) if input
+
+      # Set up the state and symbol stacks
+      @statestack = []
+      @symstack = []
+
+      pslice.stack = @symstack
+      errtoken = nil
+
+      # The start state is assumed to be (0,$end)
+      @statestack.push(0)
+      sym = YaccSymbol.new
+      sym.type = :"$end"
+      @symstack.push(sym)
+      state = 0
+
+      while true
+        # Get the next symbol on the input.  If a lookahead symbol
+        # is already set, we just use that. Otherwise, we'll pull
+        # the next token off of the lookaheadstack or from the lexer
+
+        # DBG # puts "State: #{state}"
+
+        unless lookahead
+          if lookaheadstack.empty?
+            lookahead = @lex.next
+          else
+            lookahead = lookaheadstack.pop
+          end
+          unless lookahead
+            lookahead = YaccSymbol.new()
+            lookahead.type = :"$end"
+          end
+        end
+
+        # Check the action table
+        ltype = lookahead.type
+        t = actions[state][ltype]
+
+        if t
+          if t > 0
+            # shift a symbol on the stack
+            @statestack.push(t)
+            state = t
+
+            # DBG # puts "Action : Shift and goto state #{t}"
+            
+            @symstack.push(lookahead)
+            lookahead = nil
+
+            # Decrease error count on successful shift
+            errorcount -= 1 if errorcount > 0
+            next
+          end
+
+          if t < 0
+            # reduce a symbol on the stack, emit a production
+            p = prod[-t]
+            pname = p.name
+            plen  = p.length
+
+            # Get production function
+            sym = YaccSymbol.new()
+            sym.type = pname
+            sym.value = nil
+
+            # DBG # if plen
+            # DBG # puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.value}.join(', ')}] and goto state #{-t}"
+            # DBG # else
+            # DBG # puts "Action : Reduce rule [#{p}] with [] and goto state #{-t}"
+            # DBG # end
+
+            if plen
+              targ = @symstack.pop(plen)
+              targ.insert(0, sym)
+
+              # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+              # The code enclosed in this section is duplicated
+              # below as a performance optimization.  Make sure
+              # changes get made in both locations.
+
+              pslice.slice = targ
+              
+              begin
+                # Call the grammar rule with our special slice object
+                @statestack.pop(plen)
+                instance_exec(*targ, &p.block)
+
+                # DBG # puts "Result : #{targ[0].value}"
+
+                @symstack.push(sym)
+                state = goto[@statestack[-1]][pname]
+                @statestack.push(state)
+              rescue YaccError
+                # If an error was set. Enter error recovery state
+                lookaheadstack.push(lookahead)
+                @symstack.pop # FIXME: this is definitely broken
+                @statestack.pop
+                state = @statestack[-1]
+                sym.type = :error
+                lookahead = sym
+                errorcount = self.class.error_count
+                @errorok = false
+              end
+              next
+              # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+            else
+              targ = [ sym ]
+
+              # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+              # The code enclosed in this section is duplicated
+              # below as a performance optimization.  Make sure
+              # changes get made in both locations.
+
+              pslice.slice = targ
+              
+              begin
+                # Call the grammar rule with our special slice object
+                @statestack.pop(plen)
+                pslice[0] = instance_exec(*pslice, &p.block)
+
+                # DBG # puts "Result : #{targ[0].value}"
+
+                @symstack.push(sym)
+                state = goto[@statestack[-1]][pname]
+                @statestack.push(state)
+              rescue
+                # If an error was set. Enter error recovery state
+                lookaheadstack.push(lookahead)
+                @symstack.pop # FIXME: this is definitely broken
+                @statestack.pop
+                state = @statestack[-1]
+                sym.type = :error
+                lookahead = sym
+                errorcount = error_count
+                @errorok = false
+              end
+              next
+              # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+            end
+          end
+
+          if t == 0
+            n = @symstack[-1]
+            result = n.value
+
+            # DBG # puts "Done   : Returning #{result}"
+
+            return result
+          end
+        end
+
+        if t == nil
+          # We have some kind of parsing error here.  To handle
+          # this, we are going to push the current token onto
+          # the tokenstack and replace it with an 'error' token.
+          # If there are any synchronization rules, they may
+          # catch it.
+          #
+          # In addition to pushing the error token, we call call
+          # the user defined p_error() function if this is the
+          # first syntax error.  This function is only called if
+          # errorcount == 0.
+          if errorcount == 0 || @errorok == true
+            errorcount = error_count
+            @errorok = false
+            errtoken = lookahead
+            errtoken = nil if errtoken.type == :"$end"
+
+            if self.class.error_handler
+              errok = @errok
+              token = @lex.next
+              restart = @restart
+              errtoken.lex = @lex if errtoken
+
+              tok = self.class.error_handler.call(errtoken)
+
+              if @errorok
+                # User must have done some kind of panic
+                # mode recovery on their own.  The
+                # returned token is the next lookahead
+                lookahead = tok
+                errtoken = nil
+                next
+              end
+            else
+              if errtoken
+                # if hasattr(errtoken,"lineno"): lineno = lookahead.lineno
+                # else: lineno = 0
+                #if lineno:
+                #    sys.stderr.write("yacc: Syntax error at line %d, token=%s\n" % (lineno, errtoken.type))
+                #else:
+                #    sys.stderr.write("yacc: Syntax error, token=%s" % errtoken.type)
+              else
+                # sys.stderr.write("yacc: Parse error in input. EOF\n")
+                return nil
+              end
+            end
+          else
+            errorcount = self.class.error_count
+          end
+
+          # case 1:  the @statestack only has 1 entry on it.  If we're in this state, the
+          # entire parse has been rolled back and we're completely hosed.   The token is
+          # discarded and we just keep going.
+
+          if @statestack.length <= 1 and lookahead.type != :"$end"
+            lookahead = nil
+            errtoken = nil
+            state = 0
+            # Nuke the pushback stack
+            lookaheadstack = []
+            next
+          end
+
+          # case 2: the @statestack has a couple of entries on it, but we're
+          # at the end of the file. nuke the top entry and generate an error token
+
+          # Start nuking entries on the stack
+          if lookahead.type == :"$end"
+            # Whoa. We're really hosed here. Bail out
+            return nil
+          end
+
+          if lookahead.type != :error
+            sym = @symstack[-1]
+            if sym.type == :error
+              # Hmmm. Error is on top of stack, we'll just nuke input
+              # symbol and continue
+              lookahead = nil
+              next
+            end
+            t = YaccSymbol.new
+            t.type = :error
+            # if hasattr(lookahead,"lineno"):
+            #    t.lineno = lookahead.lineno
+            t.value = lookahead
+            lookaheadstack.push(lookahead)
+            lookahead = t
+          else
+            @symstack.pop
+            @statestack.pop
+            state = @statestack[-1]       # Potential bug fix
+          end
+
+          next
+        end
+
+        # Call an error function here
+        raise RuntimeError.new("yacc: internal parser error!!!")
+      end
+    end
+
+    protected
+    def grammar
+      return @grammar if @grammar
+
+      @grammar = Grammar.new(@lex.class.terminals)
+
+      self.class.prec_rules.each do |assoc, terms|
+        terms.each_with_index do |term, i|
+          @grammar.set_precedence(term, assoc, i)
+        end
+      end
+
+      self.class.parsed_rules.each do |prod, block|
+        @grammar.add_production(*prod, &block)
+      end
+
+      @grammar.set_start
+
+      @grammar.build_lritems
+
+      @lr_table = LRTable.new(@grammar)
+
+      @lr_table.parse_table
+
+      @grammar
+    end
+
+    class << self
+      attr_accessor :rules, :grammar, :lexer_class, :prec_rules
+
+      def rule(desc, &block)
+        self.rules << [desc, block]
+      end
+
+      def lexer(&block)
+        @lexer_class = Class.new(Lex, &block)
+      end
+
+      def rules
+        @rules ||= []
+      end
+
+      def precedence(*prec)
+        assoc = prec.shift
+        self.prec_rules << [assoc, prec.reverse]
+      end
+
+      def prec_rules
+        @prec_rules ||= []
+      end
+
+      def error_count
+        3
+      end
+
+      def parsed_rules
+        @parsed_rules if @parsed_rules
+
+        @parsed_rules = []
+        rp = RuleParser.new
+        self.rules.each do |d, b|
+          rp.parse(d).each do |prod|
+            @parsed_rules << [prod, b]
+          end
+        end
+        @parsed_rules
+      end
+    end
+  end
+end