ruby_parser 3.0.0.a3 → 3.0.0.a4

data/History.txt

@@ -1,3 +1,24 @@
+=== 3.0.0.a4 / 2012-07-26
+
+* 10 minor enhancements:
+
+  * 'rake debug' defaults to 1.9 parser since that's all I'm doing these days
+  * 1.9: Fixed f { |(a, b, ...), ...| ... } handling.
+  * Added 'rake extract F=path' task to quickly extract errors from large files
+  * Added on_error handler to provide more readable error message.
+  * Aliased #process to #parse.
+  * Renamed #parse to #process (legacy name), added default path of '(string)'
+  * cleaned ruby_parse_extract_error  output and fixed to 1.9 parser
+  * ruby_parse_extract_error expands shell globs from ARGV
+  * ruby_parse_extract_error should also capture RuntimeError
+  * yyerror(msg) now warns with the message instead of ignoring it.
+
+* 3 bug fixes:
+
+  * 1.9: Fixed bug lexing/parsing [ in rhs.
+  * 1.9: Fixed f { |((a, b), c)| ... } handling
+  * 1.9: fixed newline handling during expr_value
+
 === 3.0.0.a3 / 2012-07-03
 
 * 1 major enhancement:

data/Rakefile

@@ -139,7 +139,7 @@ task :compare19 do
 end
 
 task :debug => :isolate do
-  ENV["V"] ||= "18"
+  ENV["V"] ||= "19"
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
   $: << "lib"
@@ -171,4 +171,13 @@ task :debug => :isolate do
   end
 end
 
+task :extract => :isolate do
+  ENV["V"] ||= "19"
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  file = ENV["F"] || ENV["FILE"]
+
+  ruby "-Ilib", "bin/ruby_parse_extract_error", file
+end
+
 # vim: syntax=Ruby

data/bin/ruby_parse_extract_error

@@ -26,7 +26,7 @@ class Racc::Parser
   end
 
   def retest_for_errors defs
-    d :retest => defs.size
+    warn "# retest: #{defs.size}"
 
     parser = self.class.new
 
@@ -37,12 +37,14 @@ end
 ARGV.replace ARGV.map { |path|
   if File.directory? path then
     Dir.glob File.join path, "**/*.rb"
+  elsif path =~ /\*\*/ then
+    Dir.glob path
   else
     path
   end
 }.flatten
 
-parser = RubyParser.new
+parser = Ruby19Parser.new
 
 ARGV.each do |file|
   ruby = file == "-" ? $stdin.read : File.read(file)
@@ -51,8 +53,8 @@ ARGV.each do |file|
     $stderr.print "Validating #{file}: "
     parser.process(ruby, file)
     $stderr.puts "good"
-  rescue Racc::ParseError => e
-    $stderr.puts "error: #{e.description}"
+  rescue RuntimeError, Racc::ParseError => e
+    $stderr.puts "error: #{e.message.strip}"
     $stderr.puts
 
     defs = parser.extract_defs

data/lib/ruby19_parser.rb

@@ -4858,31 +4858,31 @@ def _reduce_353(val, _values, result)
 end
 
 def _reduce_354(val, _values, result)
-                      raise "no4: #{val.inspect}"
+                      result = assignable val[0]
                     
     result
 end
 
 def _reduce_355(val, _values, result)
-                      raise "no5: #{val.inspect}"
+                      result = val[1]
                     
     result
 end
 
 def _reduce_356(val, _values, result)
-                      raise "no6: #{val.inspect}"
+                      result = s(:array, val[0])
                     
     result
 end
 
 def _reduce_357(val, _values, result)
-                      raise "no7: #{val.inspect}"
+                      result = list_append val[0], val[2]
                     
     result
 end
 
 def _reduce_358(val, _values, result)
-                      raise "no8: #{val.inspect}"
+                      result = block_var val[0], nil, nil
                     
     result
 end
@@ -5840,7 +5840,7 @@ def _reduce_523(val, _values, result)
                       identifier = val[0].to_sym
                       self.env[identifier] = :lvar
 
-                      result = val[0]
+                      result = identifier
                     
     result
 end
@@ -5848,14 +5848,21 @@ end
 # reduce 524 omitted
 
 def _reduce_525(val, _values, result)
-                      raise "no19: #{val.inspect}"
+                      result = val[1]
                     
     result
 end
 
 def _reduce_526(val, _values, result)
-                      result = s(:args)
-                      result << val[0].to_sym
+                      case val[0]
+                      when Symbol then
+                        result = s(:args)
+                        result << val[0].to_sym
+                      when Sexp then
+                        result = val[0]
+                      else
+                        raise "Unknown f_arg type: #{val.inspect}"
+                      end
                     
     result
 end

data/lib/ruby19_parser.y

@@ -1191,25 +1191,25 @@ rule
 
           f_marg: f_norm_arg
                     {
-                      raise "no4: #{val.inspect}"
+                      result = assignable val[0]
                     }
                 | tLPAREN f_margs rparen
                     {
-                      raise "no5: #{val.inspect}"
+                      result = val[1]
                     }
 
      f_marg_list: f_marg
                     {
-                      raise "no6: #{val.inspect}"
+                      result = s(:array, val[0])
                     }
                 | f_marg_list tCOMMA f_marg
                     {
-                      raise "no7: #{val.inspect}"
+                      result = list_append val[0], val[2]
                     }
 
          f_margs: f_marg_list
                     {
-                      raise "no8: #{val.inspect}"
+                      result = block_var val[0], nil, nil
                     }
                 | f_marg_list tCOMMA tSTAR f_norm_arg
                     {
@@ -1883,19 +1883,29 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       identifier = val[0].to_sym
                       self.env[identifier] = :lvar
 
-                      result = val[0]
+                      result = identifier
                     }
 
       f_arg_item: f_norm_arg
+                    # { # TODO
+                    #   result = assignable val[0]
+                    # }
                 | tLPAREN f_margs rparen
                     {
-                      raise "no19: #{val.inspect}"
+                      result = val[1]
                     }
 
            f_arg: f_arg_item
                     {
-                      result = s(:args)
-                      result << val[0].to_sym
+                      case val[0]
+                      when Symbol then
+                        result = s(:args)
+                        result << val[0].to_sym
+                      when Sexp then
+                        result = val[0]
+                      else
+                        raise "Unknown f_arg type: #{val.inspect}"
+                      end
                     }
                 | f_arg tCOMMA f_arg_item
                     {

data/lib/ruby_lexer.rb

@@ -666,6 +666,8 @@ class RubyLexer
           self.lineno = nil
           c = src.matched
           if c == '#' then
+            # TODO: add magic comment handling?
+
             src.pos -= 1
 
             while src.scan(/\s*#.*(\n+|\z)/) do
@@ -681,7 +683,7 @@ class RubyLexer
           src.scan(/\n+/)
 
           if [:expr_beg, :expr_fname,
-              :expr_dot, :expr_class].include? lex_state then
+              :expr_dot, :expr_class, :expr_value].include? lex_state then
             next
           end
 
@@ -807,7 +809,7 @@ class RubyLexer
             else
               rb_compile_error "unexpected '['"
             end
-          elsif lex_state == :expr_beg || lex_state == :expr_mid then
+          elsif is_beg? then
             self.tern.push false
             result = :tLBRACK
           elsif lex_state.is_argument && space_seen then

data/lib/ruby_parser_extras.rb

@@ -75,48 +75,10 @@ class RPStringScanner < StringScanner
       s
     end
   end
-
-  # TODO:
-  # def last_line(src)
-  #   if n = src.rindex("\n")
-  #     src[(n+1) .. -1]
-  #   else
-  #     src
-  #   end
-  # end
-  # private :last_line
-
-  # def next_words_on_error
-  #   if n = @src.rest.index("\n")
-  #     @src.rest[0 .. (n-1)]
-  #   else
-  #     @src.rest
-  #   end
-  # end
-
-  # def prev_words_on_error(ev)
-  #   pre = @pre
-  #   if ev and /#{Regexp.quote(ev)}$/ =~ pre
-  #     pre = $`
-  #   end
-  #   last_line(pre)
-  # end
-
-  # def on_error(et, ev, values)
-  #   lines_of_rest = @src.rest.to_a.length
-  #   prev_words = prev_words_on_error(ev)
-  #   at = 4 + prev_words.length
-  #   message = <<-MSG
-  # RD syntax error: line #{@blockp.line_index - lines_of_rest}:
-  # ...#{prev_words} #{(ev||'')} #{next_words_on_error()} ...
-  #   MSG
-  #   message << " " * at + "^" * (ev ? ev.length : 0) + "\n"
-  #   raise ParseError, message
-  # end
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a3' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a4' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -207,6 +169,8 @@ module RubyParserStuff
             raise "wtf? #{val.inspect}" unless lasgn[0] == :lasgn
             result << lasgn[1]
           end
+        when :lasgn then
+          result << val
         else
           raise "unhandled sexp: #{val.inspect}"
         end
@@ -240,6 +204,8 @@ module RubyParserStuff
           r.concat v[1..-1].map { |s| s(:lasgn, s) }
         when :block_arg then
           r << s(:lasgn, :"&#{v.last}")
+        when :masgn then
+          r << v
         else
           raise "block_args19 #{id} unhandled sexp type:: #{v.inspect}"
         end
@@ -263,7 +229,7 @@ module RubyParserStuff
       case r.last.first
       when :splat then
         r = s(:masgn, r)
-      when :lasgn then
+      when :lasgn, :masgn then
         r = r.last
       else
         raise "oh noes!: #{r.inspect}"
@@ -995,8 +961,17 @@ module RubyParserStuff
   end
 
   def yyerror msg
-    # for now do nothing with the msg
+    warn msg
+    super()
+  end
+
+  def on_error(et, ev, values)
     super
+  rescue Racc::ParseError => e
+    # I don't like how the exception obscures the error message
+    msg = "# ERROR: %s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
+    warn msg
+    raise
   end
 
   class Keyword
@@ -1215,11 +1190,13 @@ class RubyParser
     @p19 = Ruby19Parser.new
   end
 
-  def parse s
-    @p19.parse s
+  def process s, f = "(string)"
+    Ruby19Parser.new.process s, f
   rescue Racc::ParseError
-    @p18.parse s
+    Ruby18Parser.new.process s, f
   end
+
+  alias :parse :process
 end
 
 ############################################################

data/test/test_ruby_parser.rb

@@ -33,10 +33,16 @@ class RubyParserTestCase < ParseTreeTestCase
   end
 
   def assert_parse_error rb, emsg
-    e = assert_raises Racc::ParseError do
-      processor.parse rb
+    e = nil
+    out, err = capture_io do
+      e = assert_raises Racc::ParseError do
+        processor.parse rb
+      end
     end
 
+    assert_equal "", out
+    assert_match(/parse error on value/, err)
+
     assert_equal emsg, e.message.strip # TODO: why strip?
   end
 
@@ -47,6 +53,11 @@ class RubyParserTestCase < ParseTreeTestCase
 end
 
 module TestRubyParserShared
+  def setup
+    super
+    # p :test => [self.class, __name__]
+  end
+
   def test_attrasgn_array_lhs
     rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
     pt = s(:attrasgn,
@@ -613,6 +624,67 @@ module TestRubyParserShared
     assert_equal 3, result.if.return.line
     assert_equal 3, result.if.return.lit.line
   end
+
+  def test_bug_and
+    rb = "true and []"
+    pt = s(:and, s(:true), s(:array))
+
+    assert_parse rb, pt
+
+    rb = "true and\ntrue"
+    pt = s(:and, s(:true), s(:true))
+
+    assert_parse rb, pt
+  end
+
+  def test_bug_args
+    rb = "f { |(a, b)| d }"
+    pt = s(:iter, s(:call, nil, :f),
+           s(:masgn, s(:array, s(:lasgn, :a), s(:lasgn, :b))),
+           s(:call, nil, :d))
+
+    assert_parse rb, pt
+  end
+
+  def test_bug_args_masgn
+    rb = "f { |((a, b), c)| }"
+    pt = s(:iter,
+           s(:call, nil, :f),
+           s(:masgn,
+             s(:array,
+               s(:masgn, s(:array, s(:lasgn, :a), s(:lasgn, :b))),
+               s(:lasgn, :c))))
+
+    assert_parse rb, pt
+  end
+
+  # TODO:
+  # def test_bug_args_masgn2
+  #   rb = "f { |((a, b), c), d| }"
+  #   pt = s(:iter,
+  #          s(:call, nil, :f),
+  #          s(:masgn,
+  #            s(:array,
+  #              s(:masgn,
+  #                s(:array,
+  #                  s(:masgn,
+  #                    s(:array, s(:lasgn, :a), s(:lasgn, :b))),
+  #                  s(:lasgn, :c))),
+  #              s(:lasgn, :d))))
+  #
+  #   assert_parse rb, pt
+  # end
+
+  # TODO:
+  # def test_bug_comma
+  #   rb = "if test ?d, dir then end"
+  #   pt = s(:if,
+  #          s(:call, nil, :test, s(:lit, 100), s(:call, nil, :dir)),
+  #          nil,
+  #          nil)
+  #
+  #   assert_parse rb, pt
+  # end
 end
 
 class TestRubyParser < MiniTest::Unit::TestCase
@@ -623,7 +695,12 @@ class TestRubyParser < MiniTest::Unit::TestCase
     rb = "while false : 42 end"
     pt = s(:while, s(:false), s(:lit, 42), true)
 
-    assert_equal pt, processor.parse(rb)
+    out, err = capture_io do
+      assert_equal pt, processor.parse(rb)
+    end
+
+    assert_empty out
+    assert_match(/parse error on value .:/, err)
 
     # 1.9 only syntax
     rb = "a.()"
@@ -633,7 +710,9 @@ class TestRubyParser < MiniTest::Unit::TestCase
 
     # bad syntax
     e = assert_raises Racc::ParseError do
-      processor.parse "a.("
+      capture_io do
+        processor.parse "a.("
+      end
     end
 
     msg = "parse error on value \"(\" (tLPAREN2)"
@@ -977,4 +1056,3 @@ class TestRuby19Parser < RubyParserTestCase
   #   assert_parse rb, pt
   # end
 end
-

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 2882648795
+  hash: 1314497749
   prerelease: 6
   segments: 
   - 3
   - 0
   - 0
   - a
-  - 3
-  version: 3.0.0.a3
+  - 4
+  version: 3.0.0.a4
 platform: ruby
 authors: 
 - Ryan Davis
@@ -38,7 +38,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-07-04 00:00:00 Z
+date: 2012-07-26 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -63,11 +63,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 3
         segments: 
         - 3
-        - 0
-        version: "3.0"
+        - 2
+        version: "3.2"
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency