ruby_parser 3.13.1 → 3.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92087c0c745fee6dd8a060c3180c8254e0f23fd0d252b23f3988a3d92890ce64
-  data.tar.gz: 4ff9742257ffdf393892e293b1548587f552b6bb92fc2484bda6805c27cf1cff
+  metadata.gz: 0adcb0c06b7ee6d29360bd8dd9641dfcce867cdd7167f59bef5134c3092cb091
+  data.tar.gz: ef51691d5a0e2f09a38a0c6d8bd8d3fccdebf19bd1c1ce41a99f761112a43c4a
 SHA512:
-  metadata.gz: 8585bd8ef2918e9a31907b71dfada37f2f4d2755685ff976702bcfa90ec0137afd1b6e71bced5c631f8549188dc62169c8137eb3cf8475b393ec65174a0ac7bc
-  data.tar.gz: ead3c27d85cae963568e98c53ded6d7199da7ea781b29895eecdd54756af714b154528dc20920b3f8982519d1fd6a2f80c933489844c1d81052cc991b88172d6
+  metadata.gz: 85e6684d7a8a6207f1040ca97768d4792ba52e2f039eee5fa83047e8e5c83a7a913f98fc5384ac8b3387829078e294e22fb009163ac4eee272557cbed715849e
+  data.tar.gz: 7e996bef3411d3f63dfd8e2af388bddb16dc8a4206bd309611aeda038feb4c66041fb2b7ba5136015d4b0453e34fdebf8c1f8d408d68dccb9031514ebff6f728

data/.autotest

@@ -7,20 +7,20 @@ require 'autotest/rcov' if ENV['RCOV']
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
   at.libs << ":../../sexp_processor/dev/lib"
-  at.add_exception "unit"
-  at.add_exception "coverage"
-  at.add_exception "coverage.info"
   at.add_exception ".diff"
+  at.add_exception ".rdoc"
+  at.add_exception ".yml"
+  at.add_exception ".txt"
   at.add_exception ".output"
-  at.add_exception "lib/ruby18_parser.rb"
-  at.add_exception "lib/ruby19_parser.rb"
-  at.add_exception "lib/ruby20_parser.rb"
-  at.add_exception "lib/ruby21_parser.rb"
-  at.add_exception "lib/ruby22_parser.rb"
-  at.add_exception "lib/ruby20_parser.y"
-  at.add_exception "lib/ruby21_parser.y"
-  at.add_exception "lib/ruby22_parser.y"
-  at.add_exception "lib/ruby_lexer.rex.rb"
+  at.add_exception "bin"
+  at.add_exception "compare"
+  at.add_exception "coverage"
+  at.add_exception "coverage.info"
+  at.add_exception "misc"
+
+  Dir["lib/ruby??_parser.{rb,y}"].each do |f|
+    at.add_exception f
+  end
 
   Dir["gauntlet*"].each do |f|
     at.add_exception f
@@ -30,12 +30,12 @@ Autotest.add_hook :initialize do |at|
   at.testlib = "minitest/autorun"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
-    g = File.basename(f, ".y").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".y").gsub("_", "_?").gsub(/2\d/, '')
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
   at.add_mapping(/^lib\/.*\.yy$/) do |f, _|
-    g = File.basename(f, ".yy").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".yy").gsub("_", "_?")
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
@@ -52,27 +52,16 @@ Autotest.add_hook :initialize do |at|
     at.files_matching(/test_.*rb$/)
   end
 
-  %w(TestEnvironment TestStackState).each do |klass|
+  %w(TestEnvironment TestStackState TestValueExpr).each do |klass|
     at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
   end
 
-  %w(TestRuby18Parser TestRuby19Parser TestRuby20Parser TestRuby21Parser TestRuby22Parser TestRuby23Parser TestParseTree).each do |klass| # HACK
-    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
+  Dir["lib/ruby??_parser.rb"].each do |s|
+    n = s[/\d+/]
+    at.extra_class_map["TestRubyParserV#{n}"] = "test/test_ruby_parser.rb"
   end
 end
 
 Autotest.add_hook :run_command do |at, _|
   system "rake parser lexer DEBUG=1"
 end
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/1.9.3-p194/bin/ruby"
-  end
-end if ENV["R19"]
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/2.0.0-p195/bin/ruby"
-  end
-end if ENV["R20"]

data/History.rdoc

@@ -1,3 +1,100 @@
+=== 3.15.1 / 2021-01-10
+
+* 1 bug fix:
+
+  * Bumped ruby version to include < 4 (trunk).
+
+=== 3.15.0 / 2020-08-31
+
+* 1 major enhancement:
+
+  * Added tentative 2.7 support.
+
+* 1 minor enhancement:
+
+  * Improved ruby_parse_extract_error's handling of moving slow files out.
+
+* 22 bug fixes:
+
+  * Bumped ruby version to include 3.0 (trunk).
+  * Fix an error related to empty ensure bodies. (presidentbeef)
+  * Fix handling of bad magic encoding comment.
+  * Fixed SystemStackError when parsing a huoooge hash, caused by a splat arg.
+  * Fixed a number of errors parsing do blocks in strange edge cases.
+  * Fixed a string backslash lexing bug when the string is an invalid encoding. (nijikon, gmcgibbon)
+  * Fixed bug assigning line number to some arg nodes.
+  * Fixed bug concatinating string literals with differing encodings.
+  * Fixed bug lexing heredoc w/ nasty mix of \r\n and \n.
+  * Fixed bug lexing multiple codepoints in \u{0000 1111 2222} forms.
+  * Fixed bug setting line numbers in empty xstrings in some contexts.
+  * Fixed edge case on call w/ begin + do block as an arg.
+  * Fixed handling of UTF BOM.
+  * Fixed handling of lexer state across string interpolation braces.
+  * Fixed infinite loop when lexing backslash+cr+newline (aka dos-files)
+  * Fixed lambda + do block edge case.
+  * Fixed lexing of some ?\M... and ?\C... edge cases.
+  * Fixed more do/brace block edge case failures.
+  * Fixed parsing bug where splat was used in the middle of a list.
+  * Fixed parsing of interpolation in heredoc-like strings. (presidentbeef)
+  * Fixed parsing some esoteric edge cases in op_asgn.
+  * Fixed unicode processing in ident chars so now they better mix.
+
+=== 3.14.2 / 2020-02-06
+
+* 1 minor enhancement:
+
+  * Cleaned up call_args and removed arg_blk_pass from ruby_parser_extras.rb! Yay!
+
+=== 3.14.1 / 2019-10-29
+
+* 1 minor enhancement:
+
+  * Declared that ruby_parser supports ruby 2.2 and up.
+
+* 3 bug fixes:
+
+  * Fixed a problem with %W with a null-byte terminator. (wtf?) (spohlenz)
+  * Fixed line numbering for command (eg methods without parentheses) arguments. (mvz)
+  * Fixed lineno on new dxstrs. (presidentbeef)
+
+=== 3.14.0 / 2019-09-24
+
+* 8 minor enhancements:
+
+  * Added Sexp#line_min & new line number debugging feature.
+  * Allow bin/ruby_parse to use RP_TIMEOUT to tweak default timeout for large files & slow debugging modes.
+  * Did you know that custom inspect can't be > 65 chars? 65! Me neither. Fixed.
+  * For now, value_expr == remove_begin.
+  * Improved error messages for unterminated regexps.
+  * Moved all STR_* consts into State::Values and made them State instances.
+  * Overhauled RubyLexer::State + Values to allow for completely separate groups of States.
+  * RubyParserExtras#remove_begin removes nested begins and patches line numbers.
+
+* 22 bug fixes:
+
+  * Changed EOF signaling to return [false, false] per doco.
+  * Changed RubyParserStuff#remove_begin to keep inner line number, if any. (mvz)
+  * Differentiated between lambda call and stabby proc. Added :lambda sexp. (whitequark)
+  * Extended State#== to check equal? first as an optimization.
+  * Fixed a bug in tab indent width.
+  * Fixed a line numbering bug for assignables.
+  * Fixed a line numbering bug for defns w/ nil bodies.
+  * Fixed another missing line number, driven out by ruby_to_ruby bootstrap.
+  * Fixed dedenting squiggle heredocs if interpolation is in the mix. (mvz)
+  * Fixed differentiation of `{ |a| ... }` vs `{ |a,| ... }`. Fixes #236 (whitequark).
+  * Fixed lex_state for symbols.
+  * Fixed lex_state for tSTRING_END and tREGEXP_END.
+  * Fixed line numbers for BEGIN block. (mvz)
+  * Fixed op_asgn1 w/ command_call having array instead of arglist. (mvz)
+  * Fixed parsing of () in a command-call (eg p).
+  * Fixed remaining missing line numbers by forcing extra checks during test and blowing up. They may still be wrong (esp after heredocs, ugh), but they're there! Special thank you to mvz for pushing on this and providing tests and PRs.
+  * Fixed some lex_state versioning issues with closing braces/bracket on ruby<25.
+  * Keep carriage return escapes in heredocs. (mvz)
+  * Massive overhaul of line number code.
+  * More line number fixes and extra tests from mvz. Thank you!
+  * Parsing BEGIN block when not at top level should raise an error. (mvz)
+  * Removed irb task from Rakefile as it has been pushed up to Hoe and doubles up here.
+
 === 3.13.1 / 2019-03-25
 
 * 1 major enhancement:

data/Manifest.txt

@@ -24,6 +24,8 @@ lib/ruby25_parser.rb
 lib/ruby25_parser.y
 lib/ruby26_parser.rb
 lib/ruby26_parser.y
+lib/ruby27_parser.rb
+lib/ruby27_parser.y
 lib/ruby_lexer.rb
 lib/ruby_lexer.rex
 lib/ruby_lexer.rex.rb

data/README.rdoc

@@ -7,9 +7,9 @@ rdoc :: http://docs.seattlerb.org/ruby_parser
 == DESCRIPTION:
 
 ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-racc--which does by default use a C extension). RP's output is
-the same as ParseTree's output: s-expressions using ruby's arrays and
-base types.
+racc--which does by default use a C extension). It outputs
+s-expressions which can be manipulated and converted back to ruby via
+the ruby2ruby gem.
 
 As an example:
 

data/Rakefile

@@ -8,11 +8,12 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 Hoe.plugin :rdoc
 
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
-V2   = %w[20 21 22 23 24 25 26]
+V2   = %w[20 21 22 23 24 25 26 27]
 V2.replace [V2.last] if ENV["FAST"] # HACK
 
 Hoe.spec "ruby_parser" do
@@ -24,6 +25,8 @@ Hoe.spec "ruby_parser" do
   dependency "rake", "< 11", :developer
   dependency "oedipus_lex", "~> 2.5", :developer
 
+  require_ruby_version [">= 2.1", "< 4"]
+
   if plugin? :perforce then     # generated files
     V2.each do |n|
       self.perforce_ignore << "lib/ruby#{n}_parser.rb"
@@ -54,6 +57,8 @@ end
 
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
+task :generate => [:lexer, :parser]
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["diff.diff"] + # not all diffs. bit me too many times
@@ -73,10 +78,6 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
-task :irb => [:isolate] do
-  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
-end
-
 def (task(:phony)).timestamp
   Time.at 0
 end
@@ -107,7 +108,7 @@ def ruby_parse version
   ruby_dir  = "ruby-#{version}"
   diff      = "diff#{v}.diff"
   rp_out    = "lib/ruby#{v}_parser.output"
-  rp_y      = "lib/ruby#{v}_parser.y"
+  _rp_y     = "lib/ruby#{v}_parser.y"
   rp_y_rb   = "lib/ruby#{v}_parser.rb"
 
   c_diff    = "compare/#{diff}"
@@ -125,13 +126,22 @@ def ruby_parse version
 
   file c_parse_y => c_tarball do
     in_compare do
-      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+      extract_glob = case version
+                     when /2\.7/
+                       "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                     else
+                       "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                     end
+      system "tar yxf #{tarball} #{ruby_dir}/#{extract_glob}"
+
       Dir.chdir ruby_dir do
         if File.exist? "tool/id2token.rb" then
           sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
         else
           sh "expand parse.y > ../#{parse_y}"
         end
+
+        ruby "-pi", "-e", 'gsub(/^%define\s+api\.pure/, "%pure-parser")', "../#{parse_y}"
       end
       sh "rm -rf #{ruby_dir}"
     end
@@ -184,16 +194,17 @@ ruby_parse "2.0.0-p648"
 ruby_parse "2.1.9"
 ruby_parse "2.2.9"
 ruby_parse "2.3.8"
-ruby_parse "2.4.5"
-ruby_parse "2.5.3"
-ruby_parse "2.6.1"
+ruby_parse "2.4.9"
+ruby_parse "2.5.8"
+ruby_parse "2.6.6"
+ruby_parse "2.7.1"
 
 task :debug => :isolate do
   ENV["V"] ||= V2.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
-  $: << "lib"
+  $:.unshift "lib"
   require "ruby_parser"
   require "pp"
 
@@ -204,19 +215,21 @@ task :debug => :isolate do
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   n = ENV["BUG"]
-  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"]
+  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "bug.rb"
+  ruby = ENV["R"] || ENV["RUBY"]
+
+  if ruby then
+    file = "env"
+  else
+    ruby = File.read file
+  end
 
-  ruby = if file then
-           File.read(file)
-         else
-           file = "env"
-           ENV["R"] || ENV["RUBY"]
-         end
 
   begin
     pp parser.process(ruby, file, time)
-  rescue Racc::ParseError => e
+  rescue ArgumentError, Racc::ParseError => e
     p e
+    puts e.backtrace.join "\n  "
     ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
@@ -225,7 +238,7 @@ task :debug => :isolate do
 end
 
 task :debug3 do
-  file    = ENV["F"]
+  file    = ENV["F"] || "bug.rb"
   verbose = ENV["V"] ? "-v" : ""
   munge    = "./tools/munge.rb #{verbose}"
 
@@ -233,12 +246,17 @@ task :debug3 do
 
   ENV.delete "V"
 
+  sh "ruby -v"
   sh "ruby -y #{file} 2>&1 | #{munge} > tmp/ruby"
   sh "./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
-  sh "rake debug F=#{file} DEBUG=1 V=25 2>&1 | #{munge} > tmp/rp"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
   sh "diff -U 999 -d tmp/{rip,rp}"
 end
 
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
+end
+
 task :cmp3 do
   sh %(emacsclient --eval '(ediff-files3 "tmp/ruby" "tmp/rip" "tmp/rp")')
 end

data/bin/ruby_parse

@@ -26,6 +26,8 @@ class File
 end
 
 begin
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   ARGV.each do |file|
     rp = RubyParser.new
     loc = `wc -l #{file}`.strip.to_i
@@ -45,7 +47,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.binread(file), file)
+        r = rp.process(File.binread(file), file, time)
         pp r unless $q
         good += 1
       rescue SyntaxError => e

data/bin/ruby_parse_extract_error

@@ -104,9 +104,14 @@ rescue Timeout::Error
   warn "TIMEOUT parsing #{file}. Skipping."
 
   if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
   elsif $t then
     File.unlink file
   end

data/compare/normalize.rb

@@ -8,6 +8,10 @@ order = []
 
 def munge s
   renames = [
+             # unquote... wtf?
+             /`(.+?)'/,          proc { $1 },
+             /"'(.+?)'"/,        proc { "\"#{$1}\"" },
+
              "'='",             "tEQL",
              "'!'",             "tBANG",
              "'%'",             "tPERCENT",
@@ -100,6 +104,43 @@ def munge s
 
              "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
 
+             # 2.7 changes:
+
+             '"global variable"',          "tGVAR",
+             '"operator-assignment"',      "tOP_ASGN",
+             '"back reference"',           "tBACK_REF",
+             '"numbered reference"',       "tNTH_REF",
+             '"local variable or method"', "tIDENTIFIER",
+             '"constant"',                 "tCONSTANT",
+
+             '"(.."',                  "tBDOT2",
+             '"(..."',                 "tBDOT3",
+             '"char literal"',         "tCHAR",
+             '"literal content"',      "tSTRING_CONTENT",
+             '"string literal"',       "tSTRING_BEG",
+             '"symbol literal"',       "tSYMBEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+
+             '"float literal"',        "tFLOAT",
+             '"imaginary literal"',    "tIMAGINARY",
+             '"integer literal"',      "tINTEGER",
+             '"rational literal"',     "tRATIONAL",
+
+             '"instance variable"',  "tIVAR",
+             '"class variable"',     "tCVAR",
+             '"terminator"',         "tSTRING_END", # TODO: switch this?
+             '"method"',             "tFID",
+             '"}"',                  "tSTRING_DEND",
+
+             '"do for block"',     "kDO_BLOCK",
+             '"do for condition"', "kDO_COND",
+             '"do for lambda"',    "kDO_LAMBDA",
+
              # UGH
              "k_LINE__",       "k__LINE__",
              "k_FILE__",       "k__FILE__",
@@ -107,13 +148,12 @@ def munge s
 
              '"defined?"',     "kDEFINED",
 
-
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
 
-             /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
-             /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
             ]