rly 0.2.1 → 0.2.2

data/lib/rly.rb

@@ -1,5 +1,6 @@
 require "rly/version"
 require "rly/lex"
+require "rly/file_lex"
 require "rly/yacc"
 require "rly/parse/rule_parser"
 

data/lib/rly/file_lex.rb

@@ -0,0 +1,45 @@
+require "rly/lex"
+
+module Rly
+  class FileLex < Lex
+    def initialize(fn=nil)
+      @inputstack = []
+      push_file(fn) if fn
+    end
+    
+    def push_file(fn)
+      @inputstack.push([@input, @pos, @filename]) if @filename
+
+      @filename = fn
+      @input = open(fn).read
+      @pos = 0
+    end
+
+    def pop_file
+      (@input, @pos, @filename) = @inputstack.pop
+    end
+
+    def next
+      begin
+        tok = super
+
+        if tok
+          return tok
+        else
+          if @inputstack.empty?
+            return nil
+          else
+            pop_file
+            redo
+          end
+        end
+      end until tok
+    end
+
+    def build_token(type, value)
+      tok = LexToken.new(type, value, self, @pos, @lineno)
+      tok.location_info[:filename] = @filename
+      tok
+    end
+  end
+end

data/lib/rly/lex.rb

@@ -119,61 +119,106 @@ module Rly
     def next
       while @pos < @input.length
         if self.class.ignores_list[@input[@pos]]
-          @pos += 1
+          ignore_symbol
           next
         end
 
-        matched = false
-        self.class.tokens.each do |type, rule, block|
-          m = rule.match(@input, @pos)
-          next unless m
-          next unless m.begin(0) == @pos
-
-          tok = LexToken.new(type, m[0], self)
-
-          matched = true
-
-          tok = block.call(tok) if block
-
-          @pos = m.end(0)
-
-          return tok if tok.type
-        end
-
-        unless matched
-          if self.class.literals_list[@input[@pos]]
-            tok = LexToken.new(@input[@pos], @input[@pos], self)
+        m = self.class.token_regexps.match(@input[@pos..-1])
 
-            matched = true
+        if m && ! m[0].empty?
+          val = nil
+          type = nil
+          resolved_type = nil
+          m.names.each do |n|
+            if m[n]
+              type = n.to_sym
+              resolved_type = (n.start_with?('__anonymous_') ? nil : type)
+              val = m[n]
+              break
+            end
+          end
 
-            @pos += 1
+          if type
+            tok = build_token(resolved_type, val)
+            @pos += m.end(0)
+            tok = self.class.callables[type].call(tok) if self.class.callables[type]
 
-            return tok
+            if tok && tok.type
+              return tok
+            else
+              next
+            end
           end
         end
+        
+        if self.class.literals_list[@input[@pos]]
+          tok = build_token(@input[@pos], @input[@pos])
+          matched = true
+          @pos += 1
+          return tok
+        end
 
-        unless matched
-          if self.class.error_hander
-            pos = @pos
-            tok = LexToken.new(:error, @input[@pos], self)
-            tok = self.class.error_hander.call(tok)
-            if pos == @pos
-              raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
-            else
-              return tok if tok && tok.type
-            end
-          else
+        if self.class.error_hander
+          pos = @pos
+          tok = build_token(:error, @input[@pos])
+          tok = self.class.error_hander.call(tok)
+          if pos == @pos
             raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
+          else
+            return tok if tok && tok.type
           end
+        else
+          raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
         end
+
       end
       return nil
     end
 
+    def build_token(type, value)
+      LexToken.new(type, value, self, @pos, @lineno)
+    end
+
+    def ignore_symbol
+      @pos += 1
+    end
+
     class << self
       def terminals
-        self.tokens.map { |t,r,b| t }.compact + self.literals_list.chars.to_a
+        self.tokens.map { |t,r,b| t }.compact + self.literals_list.chars.to_a + self.metatokens_list
+      end
+
+      def callables
+        @callables ||= {}
+      end
+
+      def token_regexps
+        return @token_regexps if @token_regexps
+
+        collector = []
+        self.tokens.each do |name, rx, block|
+          name = "__anonymous_#{block.hash}".to_sym unless name
+
+          self.callables[name] = block
+          
+          rxs = rx.to_s
+          named_rxs = "\\A(?<#{name}>#{rxs})"
+
+          collector << named_rxs
+        end
+
+        rxss = collector.join('|')
+        @token_regexps = Regexp.new(rxss)
       end
+
+      def metatokens_list
+        @metatokens_list ||= []
+      end
+
+      def metatokens(*args)
+        @metatokens_list = args
+      end
+
       # Returns the list of registered tokens
       #
       # @api private

data/lib/rly/lex_token.rb

@@ -1,13 +1,14 @@
 module Rly
 
   class LexToken
-    attr_accessor :value
-    attr_reader :type, :lexer
+    attr_accessor :value, :type, :location_info
+    attr_reader :lexer
 
-    def initialize(type, value, lexer)
+    def initialize(type, value, lexer, pos=0, lineno=0, filename=nil)
       @type = type
       @value = value
       @lexer = lexer
+      @location_info = { pos: pos, lineno: lineno, filename: filename }
     end
 
     def to_s

data/lib/rly/parse/grammar.rb

@@ -45,7 +45,7 @@ module Rly
       end
 
       mapname = "#{name.to_s} -> #{symbols.to_s}"
-      raise ArgumentError if @prodmap[mapname]
+      raise ArgumentError.new("Production #{mapname} is already defined!") if @prodmap[mapname]
 
       index = @productions.count
       @nonterminals[name] = [] unless @nonterminals[name]
@@ -79,6 +79,7 @@ module Rly
     end
 
     def set_start(symbol=nil)
+      raise RuntimeError.new("No productions defined in #{self}") if @productions.empty?
       symbol = @productions[1].name unless symbol
       raise ArgumentError unless @nonterminals[symbol]
       @productions[0] = Production.new(0, :"S'", [symbol])
@@ -119,6 +120,7 @@ module Rly
       while true
         any_changes = false
         nonterminals.keys.each do |n|
+          raise RuntimeError.new("Unefined production '#{n}'") unless @prodnames[n]
           @prodnames[n].each do |p|
             _first(p.prod).each do |f|
               unless @first[n].include?(f)

data/lib/rly/parse/lr_table.rb

@@ -81,17 +81,17 @@ module Rly
                     # Need to decide on shift or reduce here
                     # By default we favor shifting. Need to add
                     # some precedence rules here.
-                    sprec, slevel = productions[st_actionp[a].number].prec
+                    sprec, slevel = productions[st_actionp[a].index].precedence
                     rprec, rlevel = precedence[a] || [:right, 0]
                     if (slevel < rlevel) || ((slevel == rlevel) && (rprec == :left))
                       # We really need to reduce here.
-                      st_action[a] = -p.number
+                      st_action[a] = -p.index
                       st_actionp[a] = p
                       if ! slevel && ! rlevel
                         log.info("  ! shift/reduce conflict for %s resolved as reduce",a)
                         @sr_conflicts << [st, a, 'reduce']
                       end
-                      productions[p.number].reduced += 1
+                      productions[p.index].reduced += 1
                     elsif (slevel == rlevel) && (rprec == :nonassoc)
                       st_action[a] = nil
                     else
@@ -105,19 +105,19 @@ module Rly
                       # Reduce/reduce conflict.   In this case, we favor the rule
                       # that was defined first in the grammar file
                       oldp = productions[-r]
-                      pp = productions[p.number]
+                      pp = productions[p.index]
                       if oldp.line > pp.line
-                        st_action[a] = -p.number
+                        st_action[a] = -p.index
                         st_actionp[a] = p
                         chosenp = pp
                         rejectp = oldp
-                        productions[p.number].reduced += 1
-                        productions[oldp.number].reduced -= 1
+                        productions[p.index].reduced += 1
+                        productions[oldp.index].reduced -= 1
                       else
                         chosenp,rejectp = oldp,pp
                       end
                       @rr_conflicts << [st, chosenp, rejectp]
-                      log.info("  ! reduce/reduce conflict for %s resolved using rule %d (%s)", a, st_actionp[a].number, st_actionp[a])
+                      log.info("  ! reduce/reduce conflict for %s resolved using rule %d (%s)", a, st_actionp[a].index, st_actionp[a])
                   else
                     raise RuntimeError("Unknown conflict in state #{st}")
                   end
@@ -128,8 +128,7 @@ module Rly
                 end
               end
             end
-          else # <-- level ok
-            # i = p.lr_index
+          else
             a = p.prod[p.lr_index+1]       # Get symbol right after the "."
             if @grammar.terminals.include?(a)
               g = lr0_goto(i, a)

data/lib/rly/parse/production.rb

@@ -27,6 +27,10 @@ module Rly
       @prod.length
     end
 
+    def line
+      @line ||= 0
+    end
+
     def lr0_added
       @lr0_added ||= 0
     end

data/lib/rly/parse/rule_parser.rb

@@ -49,6 +49,9 @@ module Rly
       @grammar.add_production(:rules, [:rule]) do |rl, r|
         rl.value = [r.value]
       end
+      @grammar.add_production(:rules, []) do |rl|
+        rl.value = [[]]
+      end
       @grammar.add_production(:rule, [:tokens]) do |r, tok|
         r.value = tok.value
       end

data/lib/rly/parse/yacc_symbol.rb

@@ -7,7 +7,7 @@ module Rly
     end
 
     def inspect
-      "#<YaccSymbol #{@type} '#{@value}'>"
+      "#<YaccSymbol #{@type} '#{to_s}'>"
     end
   end
 end

data/lib/rly/version.rb

@@ -1,4 +1,4 @@
 module Rly
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
   PLY_BASE_VERSION = "3.4"
 end

data/lib/rly/yacc.rb

@@ -22,6 +22,8 @@ module Rly
     end
 
     def parse(input=nil, trace=false)
+      @trace = trace
+
       lookahead = nil
       lookaheadstack = []
       actions = @lr_table.lr_action
@@ -56,7 +58,7 @@ module Rly
         # is already set, we just use that. Otherwise, we'll pull
         # the next token off of the lookaheadstack or from the lexer
 
-        puts "State  : #{state}" if trace
+        puts "State  : #{state}" if @trace
 
         unless lookahead
           if lookaheadstack.empty?
@@ -70,7 +72,7 @@ module Rly
           end
         end
 
-        puts "Stack  : #{(@symstack[1..-1].map{|s|s.type}.join(' ') + ' ' + lookahead.inspect).lstrip}" if trace
+        puts "Stack  : #{(@symstack[1..-1].map{|s|s.type}.join(' ') + ' ' + lookahead.inspect).lstrip}" if @trace
 
         # Check the action table
         ltype = lookahead.type
@@ -82,7 +84,7 @@ module Rly
             @statestack.push(t)
             state = t
 
-            puts "Action : Shift and goto state #{t}" if trace
+            puts "Action : Shift and goto state #{t}" if @trace
             
             @symstack.push(lookahead)
             lookahead = nil
@@ -103,9 +105,9 @@ module Rly
             sym.type = pname
             sym.value = nil
 
-            if trace
+            if @trace
               if plen
-                puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.value}.join(', ')}] and goto state #{-t}"
+                puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.inspect}.join(', ')}] and goto state #{-t}"
               else
                 puts "Action : Reduce rule [#{p}] with [] and goto state #{-t}"
               end
@@ -127,7 +129,7 @@ module Rly
                 @statestack.pop(plen)
                 instance_exec(*targ, &p.block)
 
-                puts "Result : #{targ[0].value}" if trace
+                puts "Result : #{targ[0].inspect}" if @trace
 
                 @symstack.push(sym)
                 state = goto[@statestack[-1]][pname]
@@ -160,7 +162,7 @@ module Rly
                 @statestack.pop(plen)
                 pslice[0] = instance_exec(*pslice, &p.block)
 
-                puts "Result : #{targ[0].value}" if trace
+                puts "Result : #{targ[0].value}" if @trace
 
                 @symstack.push(sym)
                 state = goto[@statestack[-1]][pname]
@@ -185,7 +187,7 @@ module Rly
             n = @symstack[-1]
             result = n.value
 
-            puts "Done   : Returning #{result}" if trace
+            puts "Done   : Returning #{result}" if @trace
 
             return result
           end
@@ -209,12 +211,7 @@ module Rly
             errtoken = nil if errtoken.type == :"$end"
 
             if self.class.error_handler
-              errok = @errok
-              token = @lex.next
-              restart = @restart
-              errtoken.lex = @lex if errtoken
-
-              tok = self.class.error_handler.call(errtoken)
+              tok = self.instance_exec(errtoken, &self.class.error_handler)
 
               if @errorok
                 # User must have done some kind of panic
@@ -226,14 +223,10 @@ module Rly
               end
             else
               if errtoken
-                # if hasattr(errtoken,"lineno"): lineno = lookahead.lineno
-                # else: lineno = 0
-                #if lineno:
-                #    sys.stderr.write("yacc: Syntax error at line %d, token=%s\n" % (lineno, errtoken.type))
-                #else:
-                #    sys.stderr.write("yacc: Syntax error, token=%s" % errtoken.type)
+                location_info = lookahead.location_info
+                puts "Fail   : Syntax error at #{location_info}, token='#{errtoken}'" if @trace
               else
-                # sys.stderr.write("yacc: Parse error in input. EOF\n")
+                puts "Fail   : Parse error in input. EOF" if @trace
                 return nil
               end
             end
@@ -361,13 +354,19 @@ module Rly
         3
       end
 
+      def on_error(lambda)
+        @error_handler = lambda
+      end
+
       def parsed_rules
         return @parsed_rules if @parsed_rules
 
         @parsed_rules = []
         rp = RuleParser.new
         self.rules.each do |desc, block|
-          rp.parse(desc).each do |(pname, p, prec)|
+          rules = rp.parse(desc)
+          raise RuntimeError.new("Failed to parse rules: #{desc}") unless rules
+          rules.each do |(pname, p, prec)|
             @parsed_rules << [pname, p, prec, block]
           end
         end

data/spec/parse/rule_parser_spec.rb

@@ -17,4 +17,15 @@ describe Rly::RuleParser do
     productions[2].should == [:expression, [:expression, '*', :expression], nil]
     productions[3].should == [:expression, [:expression, '/', :expression], nil]
   end
+
+  it "tokenizes the rule correctly" do
+    s = 'maybe_superclasses : ":" superclasses |'
+    l = Rly::RuleParser.lexer_class.new(s)
+
+    l.next.type.should == :ID
+    l.next.type.should == ':'
+    l.next.type.should == :LITERAL
+    l.next.type.should == :ID
+    l.next.type.should == '|'
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rly
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-18 00:00:00.000000000 Z
+date: 2012-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -43,6 +43,7 @@ files:
 - Rakefile
 - assets/ply_dump.erb
 - lib/rly.rb
+- lib/rly/file_lex.rb
 - lib/rly/lex.rb
 - lib/rly/lex_token.rb
 - lib/rly/parse/grammar.rb