ruby_parser 3.0.1 → 3.0.2

data/History.txt

@@ -1,3 +1,24 @@
+=== 3.0.2 / 2012-11-21
+
+52 down makes 99.9767% or 3.7σ. 130 files failed to parse out of 558k.
+
+* 4 minor enhancements:
+
+  * Added RP_TIMEOUT env var to override default timeout of 10 seconds.
+  * Minor optimization to RubyLexer#parse_number
+  * Only output parseerror output to stderr if $DEBUG.
+  * ruby_parse_extract_error modified to include 'it' blocks in its search.
+
+* 7 bug fixes:
+
+  * 1.9: Fixed args in dot-call forms (eg f.(...)).
+  * 1.9: Fixed lexing stabby lambda w/ do/end
+  * Deal better with DOS files. Ugh.
+  * Fix line number of production after heredoc.
+  * Fixed RubyParser#process to reuse parser instances across calls.
+  * Fixed line numbers for several productions.
+  * new_call sets line number to smallest line number of members.
+
 === 3.0.1 / 2012-11-02
 
 * 2 bug fixes -- both based on MRI bugs... Yay?:

data/Rakefile

@@ -144,6 +144,7 @@ task :debug => :isolate do
 
   $: << "lib"
   require 'ruby_parser'
+  require 'pp'
 
   parser = if ENV["V"] == "18" then
              Ruby18Parser.new
@@ -151,6 +152,8 @@ task :debug => :isolate do
              Ruby19Parser.new
            end
 
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   file = ENV["F"] || ENV["FILE"]
 
   ruby = if file then
@@ -161,7 +164,7 @@ task :debug => :isolate do
          end
 
   begin
-    p parser.process(ruby, file)
+    pp parser.process(ruby, file, time)
   rescue Racc::ParseError => e
     p e
     ss = parser.lexer.src
@@ -185,4 +188,8 @@ task :extract => :isolate do
   ruby "-Ilib", "bin/ruby_parse_extract_error", file
 end
 
+task :bugs do
+  sh "for f in bug*.rb ; do rake debug F=$f && rm $f ; done"
+end
+
 # vim: syntax=Ruby

data/bin/ruby_parse_extract_error

@@ -21,7 +21,7 @@ class Racc::Parser
     src = ss.string
     pre_error = src[0...ss.pos]
 
-    defs = pre_error.grep(/^ *def/)
+    defs = pre_error.grep(/^ *(?:def|it)/)
 
     raise "can't figure out where the bad code starts" unless defs.last
 
@@ -34,7 +34,7 @@ class Racc::Parser
 
     src = pre_error + post_error[0..idx+$&.length]
 
-    src.scan(/^(( *)def .*?^\2end)/m)
+    src.scan(/^(( *)(?:def|it) .*?^\2end)/m)
   end
 
   def retest_for_errors defs
@@ -56,7 +56,7 @@ def expand path
       files << f if File.file? f
     end
 
-    files
+    files.sort
   else
     Dir.glob path
   end
@@ -86,10 +86,11 @@ end
 
 def process file
   ruby = file == "-" ? $stdin.read : File.read(file)
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   $stderr.print "# Validating #{file}: "
   parser = Ruby19Parser.new
-  parser.process(ruby, file)
+  parser.process(ruby, file, time)
   warn "good"
   File.unlink file if $d
 rescue Timeout::Error

data/lib/ruby19_parser.rb

@@ -4740,13 +4740,15 @@ def _reduce_322(val, _values, result)
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno, lexer.src.beginning_of_line?
+                      result = lexer.lineno
                     
     result
 end
 
 def _reduce_323(val, _values, result)
                       result = new_defn val
+                      result[2].line val[2]
+
                       self.env.unextend
                       self.in_def = false
                     
@@ -4764,12 +4766,14 @@ def _reduce_325(val, _values, result)
                       self.in_single += 1
                       self.env.extend
                       lexer.lex_state = :expr_end # force for args
+                      result = lexer.lineno
                     
     result
 end
 
 def _reduce_326(val, _values, result)
                       result = new_defs val
+                      result[3].line val[5]
 
                       self.env.unextend
                       self.in_single -= 1
@@ -5209,13 +5213,13 @@ def _reduce_408(val, _values, result)
 end
 
 def _reduce_409(val, _values, result)
-                      result = new_call val[0], :call
+                      result = new_call val[0], :call, val[2]
                     
     result
 end
 
 def _reduce_410(val, _values, result)
-                      result = new_call val[0], :call
+                      result = new_call val[0], :call, val[2]
                     
     result
 end

data/lib/ruby19_parser.y

@@ -1109,11 +1109,13 @@ rule
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno, lexer.src.beginning_of_line?
+                      result = lexer.lineno
                     }
                     f_arglist bodystmt kEND
                     {
                       result = new_defn val
+                      result[2].line val[2]
+
                       self.env.unextend
                       self.in_def = false
                     }
@@ -1127,10 +1129,12 @@ rule
                       self.in_single += 1
                       self.env.extend
                       lexer.lex_state = :expr_end # force for args
+                      result = lexer.lineno
                     }
                     f_arglist bodystmt kEND
                     {
                       result = new_defs val
+                      result[3].line val[5]
 
                       self.env.unextend
                       self.in_single -= 1
@@ -1452,11 +1456,11 @@ rule
                     }
                 | primary_value tDOT paren_args
                     {
-                      result = new_call val[0], :call
+                      result = new_call val[0], :call, val[2]
                     }
                 | primary_value tCOLON2 paren_args
                     {
-                      result = new_call val[0], :call
+                      result = new_call val[0], :call, val[2]
                     }
                 | kSUPER paren_args
                     {

data/lib/ruby_lexer.rb

@@ -279,12 +279,12 @@ class RubyLexer
     case
     when src.scan(/[+-]?0[xXbBdD]\b/) then
       rb_compile_error "Invalid numeric format"
+    when src.scan(/[+-]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0[Dd][0-9_]+)/) then
+      int_with_base(10)
     when src.scan(/[+-]?0x[a-f0-9_]+/i) then
       int_with_base(16)
     when src.scan(/[+-]?0[Bb][01_]+/) then
       int_with_base(2)
-    when src.scan(/[+-]?0[Dd][0-9_]+/) then
-      int_with_base(10)
     when src.scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
       rb_compile_error "Illegal octal digit."
     when src.scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
@@ -298,9 +298,7 @@ class RubyLexer
       end
       self.yacc_value = number.to_f
       :tFLOAT
-    when src.scan(/[+-]?0\b/) then
-      int_with_base(10)
-    when src.scan(/[+-]?[\d_]+\b/) then
+    when src.scan(/[+-]?[0-9_]+(?![e])/) then
       int_with_base(10)
     else
       rb_compile_error "Bad number format"
@@ -1233,7 +1231,7 @@ class RubyLexer
             return process_token(command_state)
           end
         elsif src.check(/\_/) then
-          if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
+          if src.beginning_of_line? && src.scan(/\__END__(\r?\n|\Z)/) then
             self.lineno = nil
             return RubyLexer::EOF
           elsif src.scan(/\_\w*/) then
@@ -1380,13 +1378,16 @@ class RubyLexer
 
           if keyword.id0 == :kDO then
             self.command_start = true
-            return :kDO_COND  if cond.is_in_state
-            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
-            return :kDO_BLOCK if state == :expr_endarg
+
             if defined?(@hack_expects_lambda) && @hack_expects_lambda
               @hack_expects_lambda = false
               return :kDO_LAMBDA
             end
+
+            return :kDO_COND  if cond.is_in_state
+            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
+            return :kDO_BLOCK if state == :expr_endarg
+
             return :kDO
           end
 

data/lib/ruby_parser_extras.rb

@@ -78,7 +78,7 @@ class RPStringScanner < StringScanner
 
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
-    self.extra_lines_added += str.count("\n")
+    self.extra_lines_added += str.count("\n") - 1
     begin
       string[charpos, 0] = str
     rescue IndexError
@@ -108,7 +108,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.1' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.2' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -355,6 +355,8 @@ module RubyParserStuff
                end
              end
 
+    result.line(result.line - 1) if result.line and lexer.src.bol?
+
     raise "identifier #{id.inspect} is not valid" unless result
 
     result
@@ -517,7 +519,6 @@ module RubyParserStuff
 
   def new_call recv, meth, args = nil
     result = s(:call, recv, meth)
-    result.line = recv.line if recv
 
     # TODO: need a test with f(&b) to produce block_pass
     # TODO: need a test with f(&b) { } to produce warning
@@ -529,6 +530,9 @@ module RubyParserStuff
     # HACK quick hack to make this work quickly... easy to clean up above
     result.concat args[1..-1]
 
+    line = result.grep(Sexp).map(&:line).compact.min
+    result.line = line if line
+
     result
   end
 
@@ -1046,7 +1050,7 @@ module RubyParserStuff
   end
 
   def yyerror msg
-    warn msg
+    warn msg if $DEBUG
     super()
   end
 
@@ -1054,8 +1058,8 @@ module RubyParserStuff
     super
   rescue Racc::ParseError => e
     # I don't like how the exception obscures the error message
-    msg = "# ERROR: %s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
-    warn msg
+    e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
+    warn e.message if $DEBUG
     raise
   end
 
@@ -1286,9 +1290,9 @@ class RubyParser
   end
 
   def process(s, f = "(string)") # parens for emacs *sigh*
-    Ruby19Parser.new.process s, f
+    @p19.process s, f
   rescue Racc::ParseError
-    Ruby18Parser.new.process s, f
+    @p18.process s, f
   end
 
   alias :parse :process

data/test/test_ruby_parser.rb

@@ -12,6 +12,18 @@ $: << File.expand_path('~/Work/p4/zss/src/sexp_processor/dev/lib')
 
 require 'pt_testcase'
 
+class Sexp
+  alias oldeq2 ==
+  def ==(obj) # :nodoc:
+    if obj.class == self.class then
+      super and
+        (self.line.nil? or obj.line.nil? or self.line == obj.line)
+    else
+      false
+    end
+  end
+end
+
 class RubyParserTestCase < ParseTreeTestCase
   attr_accessor :result, :processor
 
@@ -41,21 +53,18 @@ class RubyParserTestCase < ParseTreeTestCase
       end
     end
 
-    assert_equal emsg, e.message.strip # TODO: why strip?
+    assert_equal emsg, e.message
   end
 
   def assert_parse_error rb, emsg
     e = nil
-    out, err = capture_io do
+    assert_silent do
       e = assert_raises Racc::ParseError do
         processor.parse rb
       end
     end
 
-    assert_equal "", out
-    assert_match(/parse error on value/, err)
-
-    assert_equal emsg, e.message.strip # TODO: why strip?
+    assert_equal emsg, e.message
   end
 
   def assert_parse_line rb, pt, line
@@ -545,21 +554,8 @@ module TestRubyParserShared
   end
 
   STARTING_LINE = {
-    "case_nested_inner_no_expr"          => 2,
-    "case_no_expr"                       => 2,
-    "case_splat"                         => 2,
-    "dstr_heredoc_expand"                => 1,
-    "dstr_heredoc_windoze_sucks"         => 1,
-    "dstr_heredoc_yet_again"             => 1,
-    "str_heredoc"                        => 1,
-    "str_heredoc_call"                   => 1,
-    "str_heredoc_empty"                  => 1,
-    "str_heredoc_indent"                 => 1,
+    "case_no_expr"                       => 2, # TODO this should be 1
     "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
-    "undef_block_1"                      => 2,
-    "undef_block_2"                      => 2,
-    "undef_block_3"                      => 2,
-    "undef_block_wtf"                    => 2,
   }
 
   def after_process_hook klass, node, data, input_name, output_name
@@ -663,9 +659,12 @@ module TestRubyParserShared
       puts string
     CODE
 
-    result = processor.parse rb
-    assert_equal 1, result.lasgn.line
-    assert_equal 4, result.call.line
+    pt = s(:block,
+           s(:lasgn, :string,
+             s(:str, "        very long string\n").line(1)).line(1),
+           s(:call, nil, :puts, s(:lvar, :string).line(4)).line(4)).line(1)
+
+    assert_parse rb, pt
   end
 
   def test_parse_line_newlines
@@ -879,6 +878,68 @@ module TestRubyParserShared
 
     assert_parse rb, pt
   end
+
+  def test_i_fucking_hate_line_numbers
+    rb = <<-EOM.gsub(/^ {6}/, '')
+      def a
+        p 1
+        a.b 2
+        c.d 3, 4
+        e.f 5
+        g.h 6, 7
+        p(1)
+        a.b(2)
+        c.d(3, 4)
+        e.f(5)
+        g.h(6, 7)
+      end
+    EOM
+
+    pt = s(:defn, :a, s(:args).line(2),
+           s(:call, nil, :p, s(:lit, 1).line(2)).line(2),
+           s(:call, s(:call, nil, :a).line(3), :b,
+             s(:lit, 2).line(3)).line(3),
+           s(:call, s(:call, nil, :c).line(4), :d,
+             s(:lit, 3).line(4), s(:lit, 4).line(4)).line(4),
+           s(:call, s(:call, nil, :e).line(5), :f,
+             s(:lit, 5).line(5)).line(5),
+           s(:call, s(:call, nil, :g).line(6), :h,
+             s(:lit, 6).line(6), s(:lit, 7).line(6)).line(6),
+           s(:call, nil, :p, s(:lit, 1).line(7)).line(7),
+           s(:call, s(:call, nil, :a).line(8), :b,
+             s(:lit, 2).line(8)).line(8),
+           s(:call, s(:call, nil, :c).line(9), :d,
+             s(:lit, 3).line(9), s(:lit, 4).line(9)).line(9),
+           s(:call, s(:call, nil, :e).line(10), :f,
+             s(:lit, 5).line(10)).line(10),
+           s(:call, s(:call, nil, :g).line(11), :h,
+             s(:lit, 6).line(11), s(:lit, 7).line(11)).line(11)
+           ).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_i_fucking_hate_line_numbers2
+    rb = <<-EOM.gsub(/^ {6}/, '')
+      def a
+          p('a')
+          b = 1
+          p b
+          c =1
+      end
+      a
+    EOM
+
+      pt = s(:block,
+             s(:defn, :a, s(:args).line(2),
+               s(:call, nil, :p, s(:str, "a").line(2)).line(2),
+               s(:lasgn, :b, s(:lit, 1).line(3)).line(3),
+               s(:call, nil, :p, s(:lvar, :b).line(4)).line(4),
+               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(1),
+             s(:call, nil, :a).line(7)).line(1)
+
+    assert_parse rb, pt
+  end
 end
 
 class TestRubyParser < MiniTest::Unit::TestCase
@@ -889,13 +950,10 @@ class TestRubyParser < MiniTest::Unit::TestCase
     rb = "while false : 42 end"
     pt = s(:while, s(:false), s(:lit, 42), true)
 
-    out, err = capture_io do
+    assert_silent do
       assert_equal pt, processor.parse(rb)
     end
 
-    assert_empty out
-    assert_match(/parse error on value .:/, err)
-
     # 1.9 only syntax
     rb = "a.()"
     pt = s(:call, s(:call, nil, :a), :call)
@@ -909,7 +967,7 @@ class TestRubyParser < MiniTest::Unit::TestCase
       end
     end
 
-    msg = "parse error on value \"(\" (tLPAREN2)"
+    msg = "(string):1 :: parse error on value \"(\" (tLPAREN2)"
     assert_equal msg, e.message.strip
   end
 end
@@ -1186,13 +1244,13 @@ class TestRuby19Parser < RubyParserTestCase
   def test_do_colon_19
     rb = "while false : 42 end"
 
-    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+    assert_parse_error rb, "(string):1 :: parse error on value \":\" (tCOLON)"
   end
 
   def test_assoc_list_19
     rb = "{1, 2, 3, 4}"
 
-    assert_parse_error rb, "parse error on value \",\" (tCOMMA)"
+    assert_parse_error rb, "(string):1 :: parse error on value \",\" (tCOMMA)"
   end
 
   def test_case_then_colon_19
@@ -1203,19 +1261,19 @@ class TestRuby19Parser < RubyParserTestCase
       end
     EOM
 
-    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+    assert_parse_error rb, "(string):2 :: parse error on value \":\" (tCOLON)"
   end
 
   def test_parse_def_xxx1
     rb = 'def f(a, *b, c = nil) end'
 
-    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+    assert_parse_error rb, '(string):1 :: parse error on value "=" (tEQL)'
   end
 
   def test_parse_def_xxx2
     rb = 'def f(a = nil, *b, c = nil) end'
 
-    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+    assert_parse_error rb, '(string):1 :: parse error on value "=" (tEQL)'
   end
 
   def test_parse_until_not_canonical
@@ -1411,7 +1469,7 @@ class TestRuby19Parser < RubyParserTestCase
   def test_motherfuckin_leading_dots2
     rb = "a\n..b"
 
-    assert_parse_error rb, 'parse error on value ".." (tDOT2)'
+    assert_parse_error rb, '(string):2 :: parse error on value ".." (tDOT2)'
   end
 
   def test_kill_me
@@ -1656,4 +1714,24 @@ class TestRuby19Parser < RubyParserTestCase
 
     assert_parse rb, pt
   end
+
+  def test_lambda_do_vs_brace
+    pt = s(:call, nil, :f, s(:iter, s(:call, nil, :lambda), 0))
+
+    rb = "f ->() {}"
+    assert_parse rb, pt
+
+    rb = "f ->() do end"
+    assert_parse rb, pt
+  end
+
+  def test_thingy
+    pt = s(:call, s(:call, nil, :f), :call, s(:lit, 42))
+
+    rb = "f.(42)"
+    assert_parse rb, pt
+
+    rb = "f::(42)"
+    assert_parse rb, pt
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 3
   prerelease: 
   segments: 
   - 3
   - 0
-  - 1
-  version: 3.0.1
+  - 2
+  version: 3.0.2
 platform: ruby
 authors: 
 - Ryan Davis
@@ -36,7 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-11-03 00:00:00 Z
+date: 2012-11-21 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -61,11 +61,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 25
+        hash: 31
         segments: 
         - 4
-        - 1
-        version: "4.1"
+        - 2
+        version: "4.2"
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -107,11 +107,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 1
         segments: 
         - 3
-        - 1
-        version: "3.1"
+        - 3
+        version: "3.3"
   type: :development
   version_requirements: *id005
 description: |-