ruby_parser 3.0.0.a6 → 3.0.0.a7

data/History.txt

@@ -1,3 +1,17 @@
+=== 3.0.0.a7 / 2012-09-21
+
+* 3 minor enhancements:
+
+  * Reorganized ruby_parse_extract_error so it will start much faster with a bunch of glob directories
+  * RubyParserStuff#process takes optional time arg and raises Timeout::Error if it goes too long. You should rescue that, ya know...
+  * ruby_parse_extract_error now checks *.rake and Rakefile on dir scan.
+
+* 3 bug fixes:
+
+  * 1.9: Fixed ternary state tracking so {a:f{f()},b:nil} won't trip up the lexer.
+  * Fixed optional values in block args (no20/no21)
+  * ruby_parse_extract_error skips non-files. Some asshats put .rb on their dirs. :(
+
 === 3.0.0.a6 / 2012-08-20
 
 * 2 minor enhancements:

data/bin/ruby_parse_extract_error

@@ -33,8 +33,6 @@ class Racc::Parser
   end
 
   def retest_for_errors defs
-    warn "# retest: #{defs.size}"
-
     parser = self.class.new
 
     parser.process(defs.join("\n\n"))
@@ -43,58 +41,62 @@ class Racc::Parser
   end
 end
 
-ARGV.replace ARGV.map { |path|
+def expand path
   if File.directory? path then
-    Dir.glob File.join path, "**/*.rb"
-  elsif path =~ /\*\*/ then
-    Dir.glob path
+    Dir.glob File.join path, "**/{*.rb,*.rake,Rakefile}"
   else
-    path
+    Dir.glob path
   end
-}.flatten.reject { |f|
-  f =~ /bad_ruby_file/ # I have intentionally bad code in my test dirs
-}
+end
 
-ARGV.each do |file|
-  ruby = file == "-" ? $stdin.read : File.read(file)
+def process_error parser
+  defs = parser.extract_defs
 
-  begin
-    $stderr.print "# Validating #{file}: "
-    parser = Ruby19Parser.new
-    parser.process(ruby, file)
-    warn "good"
-    File.unlink file if $d
-  rescue StandardError, SyntaxError, Racc::ParseError => e
-    warn "# error: #{e.message.strip}" unless $q
-    warn ""
-    next if $q
-
-    begin
-      defs = parser.extract_defs
-
-      orig_size = defs.size
-
-      if parser.retest_for_errors defs then
-        warn "Can't reproduce error with just methods, punting..."
-        next
-      end
+  orig_size = defs.size
+
+  if parser.retest_for_errors defs then
+    warn "Can't reproduce error with just methods, punting..."
+    return
+  end
 
-      mandatory = defs.pop
-
-      catch :extract_done do
-        (1..defs.size).each do |perm_size|
-          defs.combination(perm_size).each do |trial|
-            trial << mandatory
-            unless parser.retest_for_errors trial then
-              puts trial.join "\n"
-              warn "# reduced repro found!"
-              throw :extract_done
-            end
-          end
+  catch :extract_done do
+    (1..defs.size).each do |perm_size|
+      defs.combination(perm_size).each do |trial|
+        unless parser.retest_for_errors trial then
+          puts trial.join "\n"
+          throw :extract_done
         end
       end
-    rescue RuntimeError, Racc::ParseError => e
-      warn "# error: #{e.message.strip}"
     end
   end
+rescue RuntimeError, Racc::ParseError => e
+  warn "# error: #{e.message.strip}"
+end
+
+def process file
+  ruby = file == "-" ? $stdin.read : File.read(file)
+
+  $stderr.print "# Validating #{file}: "
+  parser = Ruby19Parser.new
+  parser.process(ruby, file)
+  warn "good"
+  File.unlink file if $d
+rescue StandardError, SyntaxError, Racc::ParseError => e
+  warn ""
+  warn "# error: #{e.message.strip}" unless $q
+  warn ""
+  return if $q
+
+  process_error parser
+rescue Timeout::Error
+  warn "TIMEOUT parsing #{file}. Skipping."
+end
+
+$stdout.sync = true
+
+ARGV.each do |path|
+  expand(path).each do |file|
+    next unless File.file? file # omg... why would you name a dir support.rb?
+    process file
+  end
 end

data/lib/ruby19_parser.rb

@@ -2593,7 +2593,7 @@ racc_reduce_table = [
   3, 266, :_reduce_527,
   3, 309, :_reduce_528,
   3, 310, :_reduce_529,
-  1, 267, :_reduce_530,
+  1, 267, :_reduce_none,
   3, 267, :_reduce_531,
   1, 307, :_reduce_532,
   3, 307, :_reduce_533,
@@ -5881,16 +5881,12 @@ def _reduce_528(val, _values, result)
 end
 
 def _reduce_529(val, _values, result)
-                      raise "no20: #{val.inspect}"
+                      result = self.assignable val[0], val[2]
                     
     result
 end
 
-def _reduce_530(val, _values, result)
-                      raise "no21: #{val.inspect}"
-                    
-    result
-end
+# reduce 530 omitted
 
 def _reduce_531(val, _values, result)
                       raise "no22: #{val.inspect}"

data/lib/ruby19_parser.y

@@ -1920,13 +1920,10 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
      f_block_opt: tIDENTIFIER tEQL primary_value
                     {
-                      raise "no20: #{val.inspect}"
+                      result = self.assignable val[0], val[2]
                     }
 
   f_block_optarg: f_block_opt
-                    {
-                      raise "no21: #{val.inspect}"
-                    }
                 | f_block_optarg tCOMMA f_block_opt
                     {
                       raise "no22: #{val.inspect}"

data/lib/ruby_lexer.rb

@@ -16,7 +16,7 @@ class RubyLexer
   attr_accessor :command_start
   attr_accessor :cmdarg
   attr_accessor :cond
-  attr_accessor :tern
+  attr_accessor :tern # TODO: rename ternary damnit... wtf
   attr_accessor :nest
 
   ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
@@ -711,6 +711,7 @@ class RubyLexer
         elsif src.scan(/[\]\)\}]/) then
           cond.lexpop
           cmdarg.lexpop
+          tern.lexpop
           self.lex_state = :expr_end
           self.yacc_value = src.matched
           result = {
@@ -718,7 +719,6 @@ class RubyLexer
             "]" => :tRBRACK,
             "}" => :tRCURLY
           }[src.matched]
-          self.tern.lexpop if [:tRBRACK, :tRCURLY].include?(result)
           return result
         elsif src.scan(/\.\.\.?|,|![=~]?/) then
           self.lex_state = :expr_beg
@@ -1295,6 +1295,10 @@ class RubyLexer
     is_arg? and space_seen and c !~ /\s/
   end
 
+  def is_label_possible? command_state
+    (lex_state == :expr_beg && !command_state) || is_arg?
+  end
+
   def yylex_paren19
     if is_beg? then
       result = :tLPAREN
@@ -1358,15 +1362,11 @@ class RubyLexer
       end
 
       unless self.tern.is_in_state
-        if (lex_state == :expr_beg && (ruby18 || !command_state)) ||
-            lex_state == :expr_arg ||
-            lex_state == :expr_cmdarg then
+        if is_label_possible? command_state then
           colon = src.scan(/:/)
 
-          if colon && src.peek(1) != ":"
-            src.unscan
+          if colon && src.peek(1) != ":" then
             self.lex_state = :expr_beg
-            src.scan(/:/)
             self.yacc_value = [token, src.lineno]
             return :tLABEL
           end
@@ -1413,6 +1413,9 @@ class RubyLexer
         end
       end
 
+      # TODO:
+      # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {
+
       self.lex_state =
         if is_beg? || lex_state == :expr_dot || is_arg? then
           if command_state then

data/lib/ruby_parser_extras.rb

@@ -3,6 +3,7 @@ require 'racc/parser'
 require 'sexp'
 require 'strscan'
 require 'ruby_lexer'
+require "timeout"
 
 def d o
   $stderr.puts o.inspect
@@ -78,7 +79,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a6' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a7' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -190,8 +191,8 @@ module RubyParserStuff
 
   def block_args19 val, id
     # HACK OMG THIS CODE IS SOOO UGLY! CLEAN ME
-    untested = %w[1 2 3 4 7 9 10 11 12 14]
-    raise "no block_args19 #{id}" if untested.include? id
+    untested = %w[1 2 3 4 7 9 10 12 14]
+    raise "no block_args19 #{id} #{val.inspect}" if untested.include? id
 
     r = s(:array)
 
@@ -213,6 +214,8 @@ module RubyParserStuff
           }
         when :block_arg then
           r << s(:lasgn, :"&#{v.last}")
+        when :lasgn then
+          r << s(:masgn, s(:array, v))
         when :masgn then
           r << v
         else
@@ -899,29 +902,36 @@ module RubyParserStuff
     lhs
   end
 
-  def process(str, file = "(string)")
-    raise "bad val: #{str.inspect}" unless String === str
+  ##
+  # Parse +str+ at path +file+ and return a sexp. Raises
+  # Timeout::Error if it runs for more than +time+ seconds.
 
-    str.lines.first(2).find { |s| s[/^# encoding: (.+)/, 1] }
-    encoding = $1
+  def process(str, file = "(string)", time = 10)
+    Timeout.timeout time do
+      raise "bad val: #{str.inspect}" unless String === str
 
-    str = str.dup
+      str.lines.first(2).find { |s| s[/^# encoding: (.+)/, 1] }
+      encoding = $1
 
-    if encoding then
-      if defined?(Encoding) then
-        str.force_encoding(encoding).encode! "utf-8"
-      else
-        warn "Skipping magic encoding comment"
+      str = str.dup
+
+      if encoding then
+        if defined?(Encoding) then
+          str.force_encoding(encoding).encode! "utf-8"
+        else
+          warn "Skipping magic encoding comment"
+        end
       end
-    end
 
-    self.file = file
-    self.lexer.src = str
+      self.file = file
+      self.lexer.src = str
 
-    @yydebug = ENV.has_key? 'DEBUG'
+      @yydebug = ENV.has_key? 'DEBUG'
 
-    do_parse
+      do_parse
+    end
   end
+
   alias :parse :process
 
   def remove_begin node

data/test/test_ruby_parser.rb

@@ -1189,10 +1189,40 @@ class TestRuby19Parser < RubyParserTestCase
     assert_parse rb, pt
   end
 
+  def test_block_arg_optional
+    rb = "a { |b = 1| }"
+    pt = s(:iter,
+           s(:call, nil, :a),
+           s(:masgn, s(:array, s(:lasgn, :b, s(:lit, 1)))))
+
+    assert_parse rb, pt
+  end
+
+  def test_zomg_sometimes_i_hate_this_project
+    rb = <<-RUBY
+      {
+        a: lambda { b ? c() : d },
+        e: nil,
+      }
+    RUBY
+
+    pt = s(:hash,
+           s(:lit, :a),
+           s(:iter,
+             s(:call, nil, :lambda),
+             nil,
+             s(:if, s(:call, nil, :b), s(:call, nil, :c), s(:call, nil, :d))),
+
+           s(:lit, :e),
+           s(:nil))
+
+    assert_parse rb, pt
+  end
+
   # def test_pipe_semicolon # HACK
   #   rb = "a.b do | ; c | end"
   #   pt = s(:iter, s(:call, s(:call, nil, :a), :b), 0)
-  # 
+  #
   #   assert_parse rb, pt
   # end
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 4039646929
+  hash: 1069898451
   prerelease: 6
   segments: 
   - 3
   - 0
   - 0
   - a
-  - 6
-  version: 3.0.0.a6
+  - 7
+  version: 3.0.0.a7
 platform: ruby
 authors: 
 - Ryan Davis
@@ -38,7 +38,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-08-20 00:00:00 Z
+date: 2012-09-21 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -63,11 +63,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 1
         segments: 
         - 3
-        - 2
-        version: "3.2"
+        - 3
+        version: "3.3"
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency