rly 0.2.0 → 0.2.1

data/.travis.yml

@@ -2,3 +2,4 @@ language: ruby
 rvm:
   - 1.9.3
   - rbx-19mode
+  - jruby-19mode

data/README.md

@@ -16,3 +16,9 @@ Install via rubygems
 You need to create lexer and parser classes for each grammar you want to process.
 It is commonly done by subclassing {Rly::Lex} and {Rly::Parse} classes (check the
 appropriate docs).
+
+You can also read the tutorials on the wiki:
+
+ * [Calculator Tutorial Part 1: Basic lexer](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-1:-Basic-lexer)
+ * [Calculator Tutorial Part 2: Basic parser](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-2:-Basic-parser)
+ * [Calculator Tutorial Part 3: Advanced parser](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-3:-Advanced-parser)

data/assets/ply_dump.erb

@@ -1,4 +1,4 @@
-Created by PLY version 3.4 (http://www.dabeaz.com/ply)
+Created by rly version <%= ver %>
 
 Grammar
 

data/lib/rly/lex.rb

@@ -89,6 +89,7 @@ module Rly
     #   t = lex.next # => nil
     def input(input)
       @input << input
+      nil
     end
 
     # Processes the next token in input
@@ -249,6 +250,7 @@ module Rly
         else
           raise ArgumentError
         end
+        nil
       end
 
       # Specifies a list of one-char literals
@@ -272,6 +274,7 @@ module Rly
       #   end
       def literals(lit)
         @literals = lit
+        nil
       end
 
       # Specifies a list of one-char symbols to be ignored in input
@@ -297,6 +300,7 @@ module Rly
       #   end
       def ignore(ign)
         @ignores = ign
+        nil
       end
 
       # Specifies a block that should be called on error
@@ -325,6 +329,7 @@ module Rly
       #   end
       def on_error(&block)
         @error_block = block
+        nil
       end
     end
   end

data/lib/rly/parse/grammar.rb

@@ -25,7 +25,7 @@ module Rly
       @start = nil
     end
 
-    def add_production(name, symbols, &block)
+    def add_production(name, symbols, enforced_prec=nil, &block)
       raise ArgumentError unless name.downcase == name
       raise ArgumentError if name == :error
 
@@ -36,7 +36,13 @@ module Rly
         end
       end
 
-      precedence = prec_for_rightmost_terminal(symbols)
+      if enforced_prec
+        precedence = @precedence[enforced_prec]
+        raise RuntimeError.new("Nothing known about the precedence of '#{enforced_prec}'") unless precedence
+        @used_precedence[precedence] = true
+      else
+        precedence = prec_for_rightmost_terminal(symbols)
+      end
 
       mapname = "#{name.to_s} -> #{symbols.to_s}"
       raise ArgumentError if @prodmap[mapname]

data/lib/rly/parse/ply_dump.rb

@@ -1,4 +1,6 @@
 require "rly/parse/lr_table"
+require "rly/version"
+require "erb"
 
 module Rly
 
@@ -17,7 +19,7 @@ module Rly
     def to_s
       fn = File.join(File.dirname(__FILE__), '..', '..', '..', 'assets', 'ply_dump.erb')
       e = ERB.new(open(fn).read)
-      e.result(TinyContext.new(g: @grammar, backlog: @backlog).get_binding)
+      e.result(TinyContext.new(g: @grammar, backlog: @backlog, ver: Rly::VERSION).get_binding)
     end
 
     def self.stub

data/lib/rly/parse/rule_parser.rb

@@ -8,6 +8,7 @@ module Rly
       return @lexer_class if @lexer_class
 
       @lexer_class = Class.new(Lex) do
+        token :PREC, /\%prec/
         token :ID, /[a-zA-Z_][a-zA-Z_0-9]*/
         token :LITERAL, /"."|'.'/ do |t|
           t.value = t.value[1]
@@ -25,10 +26,20 @@ module Rly
 
       @grammar = Grammar.new(self.class.lexer_class.terminals)
 
+      @grammar.add_production(:grammar_def, [:grammar]) do |d, g|
+        d.value = g.value
+      end
+      @grammar.add_production(:grammar, [:ID, ':', :rules, :PREC, :ID]) do |g, pname, _, r, _, prec|
+        productions = []
+        r.value.each do |p|
+          productions << [pname.value.to_sym, p, prec.value.to_sym]
+        end
+        g.value = productions
+      end
       @grammar.add_production(:grammar, [:ID, ':', :rules]) do |g, pname, _, r|
         productions = []
         r.value.each do |p|
-          productions << [pname.value.to_sym, p]
+          productions << [pname.value.to_sym, p, nil]
         end
         g.value = productions
       end

data/lib/rly/parse/yacc_symbol.rb

@@ -2,5 +2,12 @@ module Rly
   class YaccSymbol
     attr_accessor :type, :value, :lineno, :endlineno, :lexpos, :endlexpos
 
+    def to_s
+      @value.to_s
+    end
+
+    def inspect
+      "#<YaccSymbol #{@type} '#{@value}'>"
+    end
   end
 end

data/lib/rly/version.rb

@@ -1,4 +1,4 @@
 module Rly
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
   PLY_BASE_VERSION = "3.4"
 end

data/lib/rly/yacc.rb

@@ -2,6 +2,7 @@ require "rly/lex"
 require "rly/parse/grammar"
 require "rly/parse/yacc_production"
 require "rly/parse/yacc_symbol"
+require "rly/parse/ply_dump"
 
 module Rly
   class YaccError < RuntimeError; end
@@ -16,7 +17,11 @@ module Rly
       @grammar = grammar
     end
 
-    def parse(input=nil)
+    def inspect
+      "#<#{self.class} ...>"
+    end
+
+    def parse(input=nil, trace=false)
       lookahead = nil
       lookaheadstack = []
       actions = @lr_table.lr_action
@@ -51,7 +56,7 @@ module Rly
         # is already set, we just use that. Otherwise, we'll pull
         # the next token off of the lookaheadstack or from the lexer
 
-        # DBG # puts "State: #{state}"
+        puts "State  : #{state}" if trace
 
         unless lookahead
           if lookaheadstack.empty?
@@ -65,6 +70,8 @@ module Rly
           end
         end
 
+        puts "Stack  : #{(@symstack[1..-1].map{|s|s.type}.join(' ') + ' ' + lookahead.inspect).lstrip}" if trace
+
         # Check the action table
         ltype = lookahead.type
         t = actions[state][ltype]
@@ -75,7 +82,7 @@ module Rly
             @statestack.push(t)
             state = t
 
-            # DBG # puts "Action : Shift and goto state #{t}"
+            puts "Action : Shift and goto state #{t}" if trace
             
             @symstack.push(lookahead)
             lookahead = nil
@@ -96,11 +103,13 @@ module Rly
             sym.type = pname
             sym.value = nil
 
-            # DBG # if plen
-            # DBG # puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.value}.join(', ')}] and goto state #{-t}"
-            # DBG # else
-            # DBG # puts "Action : Reduce rule [#{p}] with [] and goto state #{-t}"
-            # DBG # end
+            if trace
+              if plen
+                puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.value}.join(', ')}] and goto state #{-t}"
+              else
+                puts "Action : Reduce rule [#{p}] with [] and goto state #{-t}"
+              end
+            end
 
             if plen
               targ = @symstack.pop(plen)
@@ -118,7 +127,7 @@ module Rly
                 @statestack.pop(plen)
                 instance_exec(*targ, &p.block)
 
-                # DBG # puts "Result : #{targ[0].value}"
+                puts "Result : #{targ[0].value}" if trace
 
                 @symstack.push(sym)
                 state = goto[@statestack[-1]][pname]
@@ -151,7 +160,7 @@ module Rly
                 @statestack.pop(plen)
                 pslice[0] = instance_exec(*pslice, &p.block)
 
-                # DBG # puts "Result : #{targ[0].value}"
+                puts "Result : #{targ[0].value}" if trace
 
                 @symstack.push(sym)
                 state = goto[@statestack[-1]][pname]
@@ -176,7 +185,7 @@ module Rly
             n = @symstack[-1]
             result = n.value
 
-            # DBG # puts "Done   : Returning #{result}"
+            puts "Done   : Returning #{result}" if trace
 
             return result
           end
@@ -194,7 +203,7 @@ module Rly
           # first syntax error.  This function is only called if
           # errorcount == 0.
           if errorcount == 0 || @errorok == true
-            errorcount = error_count
+            errorcount = self.class.error_count
             @errorok = false
             errtoken = lookahead
             errtoken = nil if errtoken.type == :"$end"
@@ -289,20 +298,26 @@ module Rly
 
       @grammar = Grammar.new(@lex.class.terminals)
 
-      self.class.prec_rules.each do |assoc, terms|
-        terms.each_with_index do |term, i|
+      self.class.prec_rules.each do |assoc, terms, i|
+        terms.each do |term|
           @grammar.set_precedence(term, assoc, i)
         end
       end
 
-      self.class.parsed_rules.each do |prod, block|
-        @grammar.add_production(*prod, &block)
+      self.class.parsed_rules.each do |pname, p, prec, block|
+        @grammar.add_production(pname, p, prec, &block)
       end
 
       @grammar.set_start
 
       @grammar.build_lritems
 
+      if self.class.store_grammar_def
+        d = PlyDump.new(@grammar)
+        gdef = d.to_s
+        open(self.class.store_grammar_def, 'w') { |f| f.write(gdef) }
+      end
+
       @lr_table = LRTable.new(@grammar)
 
       @lr_table.parse_table
@@ -311,14 +326,20 @@ module Rly
     end
 
     class << self
-      attr_accessor :rules, :grammar, :lexer_class, :prec_rules
+      attr_accessor :rules, :grammar, :lexer_class, :prec_rules, :error_handler, :store_grammar_def
+
+      def store_grammar(fn)
+        @store_grammar_def = fn
+      end
 
       def rule(desc, &block)
         self.rules << [desc, block]
+        nil
       end
 
       def lexer(&block)
         @lexer_class = Class.new(Lex, &block)
+        nil
       end
 
       def rules
@@ -327,7 +348,9 @@ module Rly
 
       def precedence(*prec)
         assoc = prec.shift
-        self.prec_rules << [assoc, prec.reverse]
+        count = self.prec_rules.length + 1
+        self.prec_rules << [assoc, prec, count]
+        nil
       end
 
       def prec_rules
@@ -339,13 +362,13 @@ module Rly
       end
 
       def parsed_rules
-        @parsed_rules if @parsed_rules
+        return @parsed_rules if @parsed_rules
 
         @parsed_rules = []
         rp = RuleParser.new
-        self.rules.each do |d, b|
-          rp.parse(d).each do |prod|
-            @parsed_rules << [prod, b]
+        self.rules.each do |desc, block|
+          rp.parse(desc).each do |(pname, p, prec)|
+            @parsed_rules << [pname, p, prec, block]
           end
         end
         @parsed_rules

data/spec/parse/calc_spec.rb

@@ -3,7 +3,7 @@ require "rly"
 module CalcSpecExample
   class CalcLex < Rly::Lex
     literals '=+-*/()'
-    ignore " \t"
+    ignore " \t\n"
 
     token :NAME, /[a-zA-Z_][a-zA-Z0-9_]*/
 
@@ -12,11 +12,10 @@ module CalcSpecExample
       t
     end
 
-    token(/\n+/) { |t| t.lexer.lineno += t.value.count("\n") }
-
     on_error do |t|
       puts "Illegal character #{t.value}"
       t.lexer.pos += 1
+      nil
     end
   end
 
@@ -44,9 +43,9 @@ module CalcSpecExample
       ex.value = e1.value.send(op.value, e2.value)
     end
 
-    # rule 'expression : "-" expression %prec UMINUS' do |ex, _, e|
-    #   ex.value = - e.value
-    # end
+    rule 'expression : "-" expression %prec UMINUS' do |ex, _, e|
+      ex.value = - e.value
+    end
 
     rule 'expression : "(" expression ")"' do |ex, _, e, _|
       ex.value = e.value
@@ -92,4 +91,8 @@ describe 'Calculator' do
     @calc.parse('magic = 42')
     @calc.parse('2 * magic').should == 84
   end
+
+  it "follows special case precedence rules" do
+    @calc.parse('2 + - 2 + 1').should == 1
+  end
 end

data/spec/parse/rule_parser_spec.rb

@@ -12,9 +12,9 @@ describe Rly::RuleParser do
     productions = p.parse(s)
 
     productions.length.should == 4
-    productions[0].should == [:expression, [:expression, '+', :expression]]
-    productions[1].should == [:expression, [:expression, '-', :expression]]
-    productions[2].should == [:expression, [:expression, '*', :expression]]
-    productions[3].should == [:expression, [:expression, '/', :expression]]
+    productions[0].should == [:expression, [:expression, '+', :expression], nil]
+    productions[1].should == [:expression, [:expression, '-', :expression], nil]
+    productions[2].should == [:expression, [:expression, '*', :expression], nil]
+    productions[3].should == [:expression, [:expression, '/', :expression], nil]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rly
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors: