ruby_parser 3.13.0 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2917a069a951276c0648eea9ab4e684111a3ce37a470356360883ddc2bd8f39
-  data.tar.gz: a0552b8faf4cb70abd400e92b536d771a6f0efc069918f0982b0405567983da3
+  metadata.gz: b1172d3be0aa30a6d959bd6ab265a6e05b391923196ec30236705637f8b05cc0
+  data.tar.gz: bedbe7c23eb256413a34d9b5dc67d975c70a8757edea6b22cebfcb82fb42d412
 SHA512:
-  metadata.gz: dda8e7842030e5fd7a3d6d180cb9dff4165acbfc0d9008a43cb00462b84aef43d4879bfd337e9732b741feba9af6bc59ef184fdbc1157170dd3d2ef6f33c5d6c
-  data.tar.gz: e975a9d0ad4e73776a153025d598141f56948f10a379fc1767d5bde90039965823c9d82397065d3005b2e8e1c2bcfe863934fe5b29946aa78a2fadeaf1ad8842
+  metadata.gz: 4e3174ada892182f6f60497feaa74668517e71bcffd44b10590f9c2fc153ba1744dbb8afd35493d1f7cb96598795e30505c922c2e53e441cbb1a7a646e9005ef
+  data.tar.gz: 2cab3beeefb45cba366c482e862bb2594169560f8920d39b28a7a86ad5c6c6582393fe91e082cd6387314f36ae5e660d8e18c2de8a4ddcefce9d40600593f795

data/.autotest

@@ -7,20 +7,20 @@ require 'autotest/rcov' if ENV['RCOV']
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
   at.libs << ":../../sexp_processor/dev/lib"
-  at.add_exception "unit"
-  at.add_exception "coverage"
-  at.add_exception "coverage.info"
   at.add_exception ".diff"
+  at.add_exception ".rdoc"
+  at.add_exception ".yml"
+  at.add_exception ".txt"
   at.add_exception ".output"
-  at.add_exception "lib/ruby18_parser.rb"
-  at.add_exception "lib/ruby19_parser.rb"
-  at.add_exception "lib/ruby20_parser.rb"
-  at.add_exception "lib/ruby21_parser.rb"
-  at.add_exception "lib/ruby22_parser.rb"
-  at.add_exception "lib/ruby20_parser.y"
-  at.add_exception "lib/ruby21_parser.y"
-  at.add_exception "lib/ruby22_parser.y"
-  at.add_exception "lib/ruby_lexer.rex.rb"
+  at.add_exception "bin"
+  at.add_exception "compare"
+  at.add_exception "coverage"
+  at.add_exception "coverage.info"
+  at.add_exception "misc"
+
+  Dir["lib/ruby??_parser.{rb,y}"].each do |f|
+    at.add_exception f
+  end
 
   Dir["gauntlet*"].each do |f|
     at.add_exception f
@@ -30,12 +30,12 @@ Autotest.add_hook :initialize do |at|
   at.testlib = "minitest/autorun"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
-    g = File.basename(f, ".y").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".y").gsub("_", "_?").gsub(/2\d/, '')
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
   at.add_mapping(/^lib\/.*\.yy$/) do |f, _|
-    g = File.basename(f, ".yy").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".yy").gsub("_", "_?")
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
@@ -52,27 +52,16 @@ Autotest.add_hook :initialize do |at|
     at.files_matching(/test_.*rb$/)
   end
 
-  %w(TestEnvironment TestStackState).each do |klass|
+  %w(TestEnvironment TestStackState TestValueExpr).each do |klass|
     at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
   end
 
-  %w(TestRuby18Parser TestRuby19Parser TestRuby20Parser TestRuby21Parser TestRuby22Parser TestRuby23Parser TestParseTree).each do |klass| # HACK
-    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
+  Dir["lib/ruby??_parser.rb"].each do |s|
+    n = s[/\d+/]
+    at.extra_class_map["TestRubyParserV#{n}"] = "test/test_ruby_parser.rb"
   end
 end
 
 Autotest.add_hook :run_command do |at, _|
   system "rake parser lexer DEBUG=1"
 end
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/1.9.3-p194/bin/ruby"
-  end
-end if ENV["R19"]
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/2.0.0-p195/bin/ruby"
-  end
-end if ENV["R20"]

data/History.rdoc

@@ -1,3 +1,126 @@
+=== 3.15.0 / 2020-08-31
+
+* 1 major enhancement:
+
+  * Added tentative 2.7 support.
+
+* 1 minor enhancement:
+
+  * Improved ruby_parse_extract_error's handling of moving slow files out.
+
+* 22 bug fixes:
+
+  * Bumped ruby version to include 3.0 (trunk).
+  * Fix an error related to empty ensure bodies. (presidentbeef)
+  * Fix handling of bad magic encoding comment.
+  * Fixed SystemStackError when parsing a huoooge hash, caused by a splat arg.
+  * Fixed a number of errors parsing do blocks in strange edge cases.
+  * Fixed a string backslash lexing bug when the string is an invalid encoding. (nijikon, gmcgibbon)
+  * Fixed bug assigning line number to some arg nodes.
+  * Fixed bug concatinating string literals with differing encodings.
+  * Fixed bug lexing heredoc w/ nasty mix of \r\n and \n.
+  * Fixed bug lexing multiple codepoints in \u{0000 1111 2222} forms.
+  * Fixed bug setting line numbers in empty xstrings in some contexts.
+  * Fixed edge case on call w/ begin + do block as an arg.
+  * Fixed handling of UTF BOM.
+  * Fixed handling of lexer state across string interpolation braces.
+  * Fixed infinite loop when lexing backslash+cr+newline (aka dos-files)
+  * Fixed lambda + do block edge case.
+  * Fixed lexing of some ?\M... and ?\C... edge cases.
+  * Fixed more do/brace block edge case failures.
+  * Fixed parsing bug where splat was used in the middle of a list.
+  * Fixed parsing of interpolation in heredoc-like strings. (presidentbeef)
+  * Fixed parsing some esoteric edge cases in op_asgn.
+  * Fixed unicode processing in ident chars so now they better mix.
+
+=== 3.14.2 / 2020-02-06
+
+* 1 minor enhancement:
+
+  * Cleaned up call_args and removed arg_blk_pass from ruby_parser_extras.rb! Yay!
+
+=== 3.14.1 / 2019-10-29
+
+* 1 minor enhancement:
+
+  * Declared that ruby_parser supports ruby 2.2 and up.
+
+* 3 bug fixes:
+
+  * Fixed a problem with %W with a null-byte terminator. (wtf?) (spohlenz)
+  * Fixed line numbering for command (eg methods without parentheses) arguments. (mvz)
+  * Fixed lineno on new dxstrs. (presidentbeef)
+
+=== 3.14.0 / 2019-09-24
+
+* 8 minor enhancements:
+
+  * Added Sexp#line_min & new line number debugging feature.
+  * Allow bin/ruby_parse to use RP_TIMEOUT to tweak default timeout for large files & slow debugging modes.
+  * Did you know that custom inspect can't be > 65 chars? 65! Me neither. Fixed.
+  * For now, value_expr == remove_begin.
+  * Improved error messages for unterminated regexps.
+  * Moved all STR_* consts into State::Values and made them State instances.
+  * Overhauled RubyLexer::State + Values to allow for completely separate groups of States.
+  * RubyParserExtras#remove_begin removes nested begins and patches line numbers.
+
+* 22 bug fixes:
+
+  * Changed EOF signaling to return [false, false] per doco.
+  * Changed RubyParserStuff#remove_begin to keep inner line number, if any. (mvz)
+  * Differentiated between lambda call and stabby proc. Added :lambda sexp. (whitequark)
+  * Extended State#== to check equal? first as an optimization.
+  * Fixed a bug in tab indent width.
+  * Fixed a line numbering bug for assignables.
+  * Fixed a line numbering bug for defns w/ nil bodies.
+  * Fixed another missing line number, driven out by ruby_to_ruby bootstrap.
+  * Fixed dedenting squiggle heredocs if interpolation is in the mix. (mvz)
+  * Fixed differentiation of `{ |a| ... }` vs `{ |a,| ... }`. Fixes #236 (whitequark).
+  * Fixed lex_state for symbols.
+  * Fixed lex_state for tSTRING_END and tREGEXP_END.
+  * Fixed line numbers for BEGIN block. (mvz)
+  * Fixed op_asgn1 w/ command_call having array instead of arglist. (mvz)
+  * Fixed parsing of () in a command-call (eg p).
+  * Fixed remaining missing line numbers by forcing extra checks during test and blowing up. They may still be wrong (esp after heredocs, ugh), but they're there! Special thank you to mvz for pushing on this and providing tests and PRs.
+  * Fixed some lex_state versioning issues with closing braces/bracket on ruby<25.
+  * Keep carriage return escapes in heredocs. (mvz)
+  * Massive overhaul of line number code.
+  * More line number fixes and extra tests from mvz. Thank you!
+  * Parsing BEGIN block when not at top level should raise an error. (mvz)
+  * Removed irb task from Rakefile as it has been pushed up to Hoe and doubles up here.
+
+=== 3.13.1 / 2019-03-25
+
+* 1 major enhancement:
+
+  * Overhauled RubyLexer#lex_state to allow for multi-state value.
+
+* 5 minor enhancements:
+
+  * Added RubyParser#current so scripts can poke around inside.
+  * Cleaned up and modernized bin/ruby_parse_extract_error.
+  * Removed RubyLexer#in_lex_state? in favor of State#=~ and #==.
+  * Renamed RubyLexer#in_arg_state? to is_after_operator?.
+  * Replaced all use of :expr_<*> symbols w/ bitfields.
+
+* 15 bug fixes:
+
+  * Added tests to show #233 already fixed.
+  * Fixed a lexer state bug dealing with complex edge case in hash  key w/ interpolated symbol and an array value. Not a great fix. (presidentbeef)
+  * Fixed a number of lex_state check problems.
+  * Fixed alias and undef keyword data.
+  * Fixed bug normalizing to utf-8 on tail of dstr (post-interpolation). (mvz)
+  * Fixed handling of hex chars for bare unicode escape. (mvz)
+  * Fixed handling of invalid string octals more gracefully. Needs generalization.
+  * Fixed lexing of regexps w/ backslashed unicode. I hope.
+  * Fixed parse_percent for EXPR_FITEM edge cases. Fixed bug #125.
+  * Fixed parsing of op_asgn on constants (eg ::X ||= 1).
+  * Fixed precedence of rescue modifier on assignments. (mvz/whitequark)
+  * Fixed some EXPR_FNAME edge cases.
+  * Fixed using unicode in gvars. Fixed bug 202. (331299)
+  * Handle invalid string literals more gracefully.
+  * Minor fix lexing %s[]
+
 === 3.13.0 / 2019-03-12
 
 * 3 major enhancements:

data/Manifest.txt

@@ -24,6 +24,8 @@ lib/ruby25_parser.rb
 lib/ruby25_parser.y
 lib/ruby26_parser.rb
 lib/ruby26_parser.y
+lib/ruby27_parser.rb
+lib/ruby27_parser.y
 lib/ruby_lexer.rb
 lib/ruby_lexer.rex
 lib/ruby_lexer.rex.rb

data/README.rdoc

@@ -7,9 +7,9 @@ rdoc :: http://docs.seattlerb.org/ruby_parser
 == DESCRIPTION:
 
 ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-racc--which does by default use a C extension). RP's output is
-the same as ParseTree's output: s-expressions using ruby's arrays and
-base types.
+racc--which does by default use a C extension). It outputs
+s-expressions which can be manipulated and converted back to ruby via
+the ruby2ruby gem.
 
 As an example:
 

data/Rakefile

@@ -8,12 +8,13 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 Hoe.plugin :rdoc
 
-Hoe.add_include_dirs File.expand_path "~/Links/SP/lib" # HACK
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
-V2   = %w[20 21 22 23 24 25 26]
+V2   = %w[20 21 22 23 24 25 26 27]
+V2.replace [V2.last] if ENV["FAST"] # HACK
 
 Hoe.spec "ruby_parser" do
   developer "Ryan Davis", "ryand-ruby@zenspider.com"
@@ -24,6 +25,8 @@ Hoe.spec "ruby_parser" do
   dependency "rake", "< 11", :developer
   dependency "oedipus_lex", "~> 2.5", :developer
 
+  require_ruby_version [">= 2.1", "< 3.1"]
+
   if plugin? :perforce then     # generated files
     V2.each do |n|
       self.perforce_ignore << "lib/ruby#{n}_parser.rb"
@@ -54,6 +57,8 @@ end
 
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
+task :generate => [:lexer, :parser]
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["diff.diff"] + # not all diffs. bit me too many times
@@ -73,10 +78,6 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
-task :irb => [:isolate] do
-  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
-end
-
 def (task(:phony)).timestamp
   Time.at 0
 end
@@ -107,7 +108,7 @@ def ruby_parse version
   ruby_dir  = "ruby-#{version}"
   diff      = "diff#{v}.diff"
   rp_out    = "lib/ruby#{v}_parser.output"
-  rp_y      = "lib/ruby#{v}_parser.y"
+  _rp_y     = "lib/ruby#{v}_parser.y"
   rp_y_rb   = "lib/ruby#{v}_parser.rb"
 
   c_diff    = "compare/#{diff}"
@@ -125,13 +126,22 @@ def ruby_parse version
 
   file c_parse_y => c_tarball do
     in_compare do
-      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+      extract_glob = case version
+                     when /2\.7/
+                       "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                     else
+                       "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                     end
+      system "tar yxf #{tarball} #{ruby_dir}/#{extract_glob}"
+
       Dir.chdir ruby_dir do
         if File.exist? "tool/id2token.rb" then
           sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
         else
           sh "expand parse.y > ../#{parse_y}"
         end
+
+        ruby "-pi", "-e", 'gsub(/^%define\s+api\.pure/, "%pure-parser")', "../#{parse_y}"
       end
       sh "rm -rf #{ruby_dir}"
     end
@@ -180,22 +190,21 @@ def ruby_parse version
   end
 end
 
-# ruby_parse "1.8.7-p374"
-# ruby_parse "1.9.3-p551"
 ruby_parse "2.0.0-p648"
 ruby_parse "2.1.9"
 ruby_parse "2.2.9"
 ruby_parse "2.3.8"
-ruby_parse "2.4.5"
-ruby_parse "2.5.3"
-ruby_parse "2.6.1"
+ruby_parse "2.4.9"
+ruby_parse "2.5.8"
+ruby_parse "2.6.6"
+ruby_parse "2.7.1"
 
 task :debug => :isolate do
   ENV["V"] ||= V2.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
-  $: << "lib"
+  $:.unshift "lib"
   require "ruby_parser"
   require "pp"
 
@@ -206,19 +215,21 @@ task :debug => :isolate do
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   n = ENV["BUG"]
-  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"]
+  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "bug.rb"
+  ruby = ENV["R"] || ENV["RUBY"]
+
+  if ruby then
+    file = "env"
+  else
+    ruby = File.read file
+  end
 
-  ruby = if file then
-           File.read(file)
-         else
-           file = "env"
-           ENV["R"] || ENV["RUBY"]
-         end
 
   begin
     pp parser.process(ruby, file, time)
-  rescue Racc::ParseError => e
+  rescue ArgumentError, Racc::ParseError => e
     p e
+    puts e.backtrace.join "\n  "
     ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
@@ -227,7 +238,7 @@ task :debug => :isolate do
 end
 
 task :debug3 do
-  file    = ENV["F"]
+  file    = ENV["F"] || "bug.rb"
   verbose = ENV["V"] ? "-v" : ""
   munge    = "./tools/munge.rb #{verbose}"
 
@@ -235,9 +246,15 @@ task :debug3 do
 
   ENV.delete "V"
 
+  sh "ruby -v"
   sh "ruby -y #{file} 2>&1 | #{munge} > tmp/ruby"
   sh "./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
-  sh "rake debug F=#{file} DEBUG=1 V=25 2>&1 | #{munge} > tmp/rp"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
+  sh "diff -U 999 -d tmp/{rip,rp}"
+end
+
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
 end
 
 task :cmp3 do

data/bin/ruby_parse

@@ -26,6 +26,8 @@ class File
 end
 
 begin
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   ARGV.each do |file|
     rp = RubyParser.new
     loc = `wc -l #{file}`.strip.to_i
@@ -45,7 +47,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.binread(file), file)
+        r = rp.process(File.binread(file), file, time)
         pp r unless $q
         good += 1
       rescue SyntaxError => e

data/bin/ruby_parse_extract_error

@@ -1,44 +1,20 @@
 #!/usr/bin/ruby -ws
 
-$d ||= false
-$d ||= ENV["DELETE"]
-$t ||= false
-$t ||= ENV["DELETE_TIMEOUT"]
-$m ||= false
-$m ||= ENV["MOVE_TIMEOUT"]
-$q ||= false
-$q ||= ENV["QUIET"]
-$v ||= ENV["V"] || "20"
-$s ||= ENV["SPEED"] || false
+$d ||= ENV["DELETE"]         || false
+$t ||= ENV["DELETE_TIMEOUT"] || false
+$m ||= ENV["MOVE_TIMEOUT"]   || false
+$q ||= ENV["QUIET"]          || false
+$s ||= ENV["SPEED"]          || false
 
 require 'rubygems'
 require 'ruby_parser'
 require 'fileutils'
 
-$parser_class = case $v
-                when "18" then
-                  Ruby18Parser
-                when "19" then
-                  Ruby19Parser
-                when "20" then
-                  Ruby20Parser
-                else
-                  abort "Unknown version #{$v.inspect}. Needs to be 18, 19, or 20"
-                end
-
-class IO
-  RUBY19 = "<3".respond_to? :encoding
-
-  class << self
-    alias :binread :read unless RUBY19
-  end
-end
-
 ARGV.push "-" if ARGV.empty?
 
-class Racc::Parser
+class RubyParser
   def extract_defs
-    ss = lexer.ss
+    ss = current.lexer.ss
 
     raise "can't access source. possible encoding issue" unless ss
 
@@ -106,6 +82,7 @@ def process_error parser
   end
 rescue RuntimeError, Racc::ParseError => e
   warn "# process error: #{e.message.strip}"
+  warn "#   #{e.backtrace.first}"
 end
 
 def process file
@@ -113,7 +90,7 @@ def process file
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   $stderr.print "# Validating #{file}: "
-  parser = $parser_class.new
+  parser = RubyParser.new
   t0 = Time.now if $s
   parser.process(ruby, file, time)
   if $s then
@@ -127,9 +104,14 @@ rescue Timeout::Error
   warn "TIMEOUT parsing #{file}. Skipping."
 
   if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
   elsif $t then
     File.unlink file
   end
@@ -137,6 +119,7 @@ rescue StandardError, SyntaxError, Racc::ParseError => e
   $exit = 1
   warn ""
   warn "# error: #{e.message.strip}" unless $q
+  warn "#   #{e.backtrace.first}"
   warn ""
   return if $q