ruby_parser 3.12.0 → 3.18.1

data/Rakefile

@@ -1,7 +1,5 @@
 # -*- ruby -*-
 
-$:.unshift "../../hoe/dev/lib"
-
 require "rubygems"
 require "hoe"
 
@@ -10,29 +8,43 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 Hoe.plugin :rdoc
 
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
-V1   = %w[18 19]
-V2   = %w[20 21 22 23 24 25]
-V1_2 = V1 + V2
+V2   = %w[20 21 22 23 24 25 26 27]
+V3   = %w[30]
+
+VERS = V2 + V3
+
+ENV["FAST"] = VERS.last if ENV["FAST"] && !VERS.include?(ENV["FAST"])
+VERS.replace [ENV["FAST"]] if ENV["FAST"]
 
 Hoe.spec "ruby_parser" do
   developer "Ryan Davis", "ryand-ruby@zenspider.com"
 
   license "MIT"
 
-  dependency "sexp_processor", "~> 4.9"
-  dependency "rake", "< 11", :developer
-  dependency "oedipus_lex", "~> 2.5", :developer
+  dependency "sexp_processor", "~> 4.16"
+  dependency "rake", [">= 10", "< 15"], :developer
+  dependency "oedipus_lex", "~> 2.6", :developer
+
+  # NOTE: Ryan!!! Stop trying to fix this dependency! Isolate just
+  # can't handle having a faux-gem half-installed! Stop! Just `gem
+  # install racc` and move on. Revisit this ONLY once racc-compiler
+  # gets split out.
+
+  dependency "racc", "~> 1.5", :developer
+
+  require_ruby_version [">= 2.1", "< 4"]
 
   if plugin? :perforce then     # generated files
-    V1_2.each do |n|
+    VERS.each do |n|
       self.perforce_ignore << "lib/ruby#{n}_parser.rb"
     end
 
-    V2.each do |n|
+    VERS.each do |n|
       self.perforce_ignore << "lib/ruby#{n}_parser.y"
     end
 
@@ -46,25 +58,68 @@ Hoe.spec "ruby_parser" do
   end
 end
 
+def maybe_add_to_top path, string
+  file = File.read path
+
+  return if file.start_with? string
+
+  warn "Altering top of #{path}"
+  tmp_path = "#{path}.tmp"
+  File.open(tmp_path, "w") do |f|
+    f.puts string
+    f.puts
+
+    f.write file
+    # TODO: make this deal with encoding comments properly?
+  end
+  File.rename tmp_path, path
+end
+
+def unifdef?
+  @unifdef ||= system("which unifdef") or abort <<~EOM
+    unifdef not found!
+
+    Please install 'unifdef' package on your system or `rake generate` on a mac.
+  EOM
+end
+
 V2.each do |n|
   file "lib/ruby#{n}_parser.y" => "lib/ruby_parser.yy" do |t|
+    unifdef?
     cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
     sh cmd
   end
+
+  file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
 end
 
-V1_2.each do |n|
+V3.each do |n|
+  file "lib/ruby#{n}_parser.y" => "lib/ruby3_parser.yy" do |t|
+    unifdef?
+    cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
+    sh cmd
+  end
+
   file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
 end
 
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
+task :parser do |t|
+  t.prerequisite_tasks.grep(Rake::FileTask).select(&:already_invoked).each do |f|
+    maybe_add_to_top f.name, "# frozen_string_literal: true"
+  end
+end
+
+task :generate => [:lexer, :parser]
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["diff.diff"] + # not all diffs. bit me too many times
         Dir["coverage.info"] +
         Dir["coverage"] +
         Dir["lib/ruby2*_parser.y"] +
+        Dir["lib/ruby3*_parser.y"] +
         Dir["lib/*.output"])
 end
 
@@ -78,10 +133,6 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
-task :irb => [:isolate] do
-  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
-end
-
 def (task(:phony)).timestamp
   Time.at 0
 end
@@ -96,7 +147,7 @@ end
 
 def dl v
   dir = v[/^\d+\.\d+/]
-  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.bz2"
+  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.xz"
   path = File.basename url
   unless File.exist? path then
     system "curl -O #{url}"
@@ -108,48 +159,68 @@ def ruby_parse version
   rp_txt    = "rp#{v}.txt"
   mri_txt   = "mri#{v}.txt"
   parse_y   = "parse#{v}.y"
-  tarball   = "ruby-#{version}.tar.bz2"
+  tarball   = "ruby-#{version}.tar.xz"
   ruby_dir  = "ruby-#{version}"
   diff      = "diff#{v}.diff"
   rp_out    = "lib/ruby#{v}_parser.output"
+  _rp_y     = "lib/ruby#{v}_parser.y"
+  rp_y_rb   = "lib/ruby#{v}_parser.rb"
 
   c_diff    = "compare/#{diff}"
   c_rp_txt  = "compare/#{rp_txt}"
   c_mri_txt = "compare/#{mri_txt}"
   c_parse_y = "compare/#{parse_y}"
   c_tarball = "compare/#{tarball}"
+  normalize = "compare/normalize.rb"
 
-  file tarball do
+  file c_tarball do
     in_compare do
       dl version
     end
   end
 
+  desc "fetch all tarballs"
+  task :fetch => c_tarball
+
   file c_parse_y => c_tarball do
     in_compare do
-      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+      extract_glob = case version
+                     when /2\.7|3\.0/
+                       "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                     else
+                       "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                     end
+      system "tar Jxf #{tarball} #{ruby_dir}/#{extract_glob}"
+
       Dir.chdir ruby_dir do
         if File.exist? "tool/id2token.rb" then
-          sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y > ../#{parse_y}"
+          sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
         else
-          cp "parse.y", "../#{parse_y}"
+          sh "expand parse.y > ../#{parse_y}"
         end
+
+        ruby "-pi", "-e", 'gsub(/^%pure-parser/, "%define api.pure")', "../#{parse_y}"
       end
       sh "rm -rf #{ruby_dir}"
     end
   end
 
-  file c_mri_txt => c_parse_y do
+  bison = Dir["/opt/homebrew/opt/bison/bin/bison",
+              "/usr/local/opt/bison/bin/bison",
+              `which bison`.chomp,
+             ].first
+
+  file c_mri_txt => [c_parse_y, normalize] do
     in_compare do
-      sh "bison -r all #{parse_y}"
+      sh "#{bison} -r all #{parse_y}"
       sh "./normalize.rb parse#{v}.output > #{mri_txt}"
       rm ["parse#{v}.output", "parse#{v}.tab.c"]
     end
   end
 
-  file rp_out => :parser
+  file rp_out => rp_y_rb
 
-  file c_rp_txt => rp_out do
+  file c_rp_txt => [rp_out, normalize] do
     in_compare do
       sh "./normalize.rb ../#{rp_out} > #{rp_txt}"
     end
@@ -160,9 +231,9 @@ def ruby_parse version
   desc "Compare all grammars to MRI"
   task :compare => compare
 
-  task c_diff => [c_mri_txt, c_rp_txt] do
+  file c_diff => [c_mri_txt, c_rp_txt] do
     in_compare do
-      system "diff -du #{mri_txt} #{rp_txt} > #{diff}"
+      sh "diff -du #{mri_txt} #{rp_txt} > #{diff}; true"
     end
   end
 
@@ -174,28 +245,62 @@ def ruby_parse version
   end
 
   task :clean do
-    rm_f Dir[c_parse_y, c_mri_txt, c_rp_txt]
+    rm_f Dir[c_mri_txt, c_rp_txt]
   end
 
   task :realclean do
-    rm_f Dir[tarball]
+    rm_f Dir[c_parse_y, tarball]
+  end
+end
+
+task :versions do
+  require "open-uri"
+  require "net/http" # avoid require issues in threads
+  require "net/https"
+
+  versions = %w[ 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 ]
+
+  base_url = "https://cache.ruby-lang.org/pub/ruby"
+
+  class Array
+    def human_sort
+      sort_by { |item| item.to_s.split(/(\d+)/).map { |e| [e.to_i, e] } }
+    end
   end
+
+  versions = versions.map { |ver|
+    Thread.new {
+      URI
+        .parse("#{base_url}/#{ver}/")
+        .read
+        .scan(/ruby-\d+\.\d+\.\d+[-\w.]*?.tar.gz/)
+        .reject { |s| s =~ /-(?:rc|preview)\d/ }
+        .human_sort
+        .last
+        .delete_prefix("ruby-")
+        .delete_suffix ".tar.gz"
+    }
+  }.map(&:value).sort
+
+  puts versions.map { |v| "ruby_parse %p" % [v] }
 end
 
-ruby_parse "1.8.7-p374"
-ruby_parse "1.9.3-p551"
 ruby_parse "2.0.0-p648"
-ruby_parse "2.1.9"
-ruby_parse "2.2.6"
-ruby_parse "2.3.3"
-# TODO ruby_parse "2.4.0"
+ruby_parse "2.1.10"
+ruby_parse "2.2.10"
+ruby_parse "2.3.8"
+ruby_parse "2.4.10"
+ruby_parse "2.5.9"
+ruby_parse "2.6.8"
+ruby_parse "2.7.4"
+ruby_parse "3.0.2"
 
 task :debug => :isolate do
-  ENV["V"] ||= V1_2.last
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
-  $: << "lib"
+  $:.unshift "lib"
   require "ruby_parser"
   require "pp"
 
@@ -206,19 +311,21 @@ task :debug => :isolate do
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   n = ENV["BUG"]
-  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"]
+  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "debug.rb"
+  ruby = ENV["R"] || ENV["RUBY"]
+
+  if ruby then
+    file = "env"
+  else
+    ruby = File.read file
+  end
 
-  ruby = if file then
-           File.read(file)
-         else
-           file = "env"
-           ENV["R"] || ENV["RUBY"]
-         end
 
   begin
     pp parser.process(ruby, file, time)
-  rescue Racc::ParseError => e
+  rescue ArgumentError, Racc::ParseError => e
     p e
+    puts e.backtrace.join "\n  "
     ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
@@ -226,22 +333,53 @@ task :debug => :isolate do
   end
 end
 
-task :debug_ruby do
-  file = ENV["F"] || ENV["FILE"]
-  sh "/Users/ryan/Desktop/DVDs/debugparser/miniruby -cwy #{file} 2>&1 | ./yuck.rb"
+task :debug3 do
+  file    = ENV["F"] || "debug.rb"
+  version = ENV["V"] || ""
+  verbose = ENV["VERBOSE"] ? "-v" : ""
+  munge    = "./tools/munge.rb #{verbose}"
+
+  abort "Need a file to parse, via: F=path.rb" unless file
+
+  ENV.delete "V"
+
+  ruby = "ruby#{version}"
+
+  sh "#{ruby} -v"
+  sh "#{ruby} -y #{file} 2>&1 | #{munge} > tmp/ruby"
+  sh "#{ruby} ./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
+  sh "diff -U 999 -d tmp/{ruby,rp}"
+end
+
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
+end
+
+task :cmp3 do
+  sh %(emacsclient --eval '(ediff-files3 "tmp/ruby" "tmp/rip" "tmp/rp")')
 end
 
 task :extract => :isolate do
-  ENV["V"] ||= V1_2.last
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
-  file = ENV["F"] || ENV["FILE"]
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
 
   ruby "-Ilib", "bin/ruby_parse_extract_error", file
 end
 
+task :parse => :isolate do
+  ENV["V"] ||= VERS.last
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
+
+  ruby "-Ilib", "bin/ruby_parse", file
+end
+
 task :bugs do
-  sh "for f in bug*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
+  sh "for f in bug*.rb bad*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
 end
 
 # vim: syntax=Ruby

data/bin/ruby_parse

@@ -26,6 +26,8 @@ class File
 end
 
 begin
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   ARGV.each do |file|
     rp = RubyParser.new
     loc = `wc -l #{file}`.strip.to_i
@@ -45,7 +47,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.binread(file), file)
+        r = rp.process(File.binread(file), file, time)
         pp r unless $q
         good += 1
       rescue SyntaxError => e

data/bin/ruby_parse_extract_error

@@ -1,51 +1,27 @@
 #!/usr/bin/ruby -ws
 
-$d ||= false
-$d ||= ENV["DELETE"]
-$t ||= false
-$t ||= ENV["DELETE_TIMEOUT"]
-$m ||= false
-$m ||= ENV["MOVE_TIMEOUT"]
-$q ||= false
-$q ||= ENV["QUIET"]
-$v ||= ENV["V"] || "20"
-$s ||= ENV["SPEED"] || false
+$d ||= ENV["DELETE"]         || false
+$t ||= ENV["DELETE_TIMEOUT"] || false
+$m ||= ENV["MOVE_TIMEOUT"]   || false
+$q ||= ENV["QUIET"]          || false
+$s ||= ENV["SPEED"]          || false
 
 require 'rubygems'
 require 'ruby_parser'
 require 'fileutils'
 
-$parser_class = case $v
-                when "18" then
-                  Ruby18Parser
-                when "19" then
-                  Ruby19Parser
-                when "20" then
-                  Ruby20Parser
-                else
-                  abort "Unknown version #{$v.inspect}. Needs to be 18, 19, or 20"
-                end
-
-class IO
-  RUBY19 = "<3".respond_to? :encoding
-
-  class << self
-    alias :binread :read unless RUBY19
-  end
-end
-
 ARGV.push "-" if ARGV.empty?
 
-class Racc::Parser
+class RubyParser
   def extract_defs
-    ss = lexer.ss
+    ss = current.lexer.ss
 
     raise "can't access source. possible encoding issue" unless ss
 
     src = ss.string
     pre_error = src[0...ss.pos]
 
-    defs = pre_error.grep(/^ *(?:def|it)/)
+    defs = pre_error.lines.grep(/^ *(?:def|it)/)
 
     raise "can't figure out where the bad code starts" unless defs.last
 
@@ -106,6 +82,7 @@ def process_error parser
   end
 rescue RuntimeError, Racc::ParseError => e
   warn "# process error: #{e.message.strip}"
+  warn "#   #{e.backtrace.first}"
 end
 
 def process file
@@ -113,7 +90,7 @@ def process file
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   $stderr.print "# Validating #{file}: "
-  parser = $parser_class.new
+  parser = RubyParser.new
   t0 = Time.now if $s
   parser.process(ruby, file, time)
   if $s then
@@ -127,9 +104,14 @@ rescue Timeout::Error
   warn "TIMEOUT parsing #{file}. Skipping."
 
   if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
   elsif $t then
     File.unlink file
   end
@@ -137,6 +119,7 @@ rescue StandardError, SyntaxError, Racc::ParseError => e
   $exit = 1
   warn ""
   warn "# error: #{e.message.strip}" unless $q
+  warn "#   #{e.backtrace.first}"
   warn ""
   return if $q
 

data/compare/normalize.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby -w
+#!/usr/bin/env ruby -w
 
 good = false
 
@@ -8,6 +8,10 @@ order = []
 
 def munge s
   renames = [
+             # unquote... wtf?
+             /`(.+?)'/,          proc { $1 },
+             /"'(.+?)'"/,        proc { "\"#{$1}\"" },
+
              "'='",             "tEQL",
              "'!'",             "tBANG",
              "'%'",             "tPERCENT",
@@ -53,6 +57,7 @@ def munge s
 
              '"&"',              "tAMPER",
              '"&&"',             "tANDOP",
+             '"&."',             "tLONELY",
              '"||"',             "tOROP",
 
              '"..."',            "tDOT3",
@@ -79,8 +84,10 @@ def munge s
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
+             "%empty",          "none", # newer bison
              "/* empty */",     "none",
              /^\s*$/,           "none",
+
              "keyword_BEGIN",   "klBEGIN",
              "keyword_END",     "klEND",
              /keyword_(\w+)/,   proc { "k#{$1.upcase}" },
@@ -88,7 +95,72 @@ def munge s
              /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
              "kVARIABLE",       "keyword_variable", # ugh
 
-             /@(\d+)\s+/,       "",
+             # 2.6 collapses klBEGIN to kBEGIN
+             "klBEGIN",   "kBEGIN",
+             "klEND",     "kEND",
+
+             /keyword_(\w+)/,          proc { "k#{$1.upcase}" },
+             /\bk_([^_][a-z_]+)/,      proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,         proc { "k#{$1.upcase}_MOD" },
+
+             "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
+
+             # 2.7 changes:
+
+             '"global variable"',          "tGVAR",
+             '"operator-assignment"',      "tOP_ASGN",
+             '"back reference"',           "tBACK_REF",
+             '"numbered reference"',       "tNTH_REF",
+             '"local variable or method"', "tIDENTIFIER",
+             '"constant"',                 "tCONSTANT",
+
+             '"(.."',                  "tBDOT2",
+             '"(..."',                 "tBDOT3",
+             '"char literal"',         "tCHAR",
+             '"literal content"',      "tSTRING_CONTENT",
+             '"string literal"',       "tSTRING_BEG",
+             '"symbol literal"',       "tSYMBEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+
+             '"float literal"',        "tFLOAT",
+             '"imaginary literal"',    "tIMAGINARY",
+             '"integer literal"',      "tINTEGER",
+             '"rational literal"',     "tRATIONAL",
+
+             '"instance variable"',  "tIVAR",
+             '"class variable"',     "tCVAR",
+             '"terminator"',         "tSTRING_END", # TODO: switch this?
+             '"method"',             "tFID",
+             '"}"',                  "tSTRING_DEND",
+
+             '"do for block"',     "kDO_BLOCK",
+             '"do for condition"', "kDO_COND",
+             '"do for lambda"',    "kDO_LAMBDA",
+             "tLABEL",             "kLABEL",
+
+             # UGH
+             "k_LINE__",       "k__LINE__",
+             "k_FILE__",       "k__FILE__",
+             "k_ENCODING__",   "k__ENCODING__",
+
+             '"defined?"',     "kDEFINED",
+
+             '"do (for condition)"', "kDO_COND",
+             '"do (for lambda)"',    "kDO_LAMBDA",
+             '"do (for block)"',     "kDO_BLOCK",
+
+             /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
+
+             /\$?@(\d+)(\s+|$)/,    "", # newer bison
+
+             # TODO: remove for 3.0 work:
+             "lex_ctxt ", "" # 3.0 production that's mostly noise right now
             ]
 
   renames.each_slice(2) do |(a, b)|
@@ -107,7 +179,7 @@ ARGF.each_line do |line|
 
   case line.strip
   when /^$/ then
-  when /^(\d+) (\$?\w+): (.*)/ then    # yacc
+  when /^(\d+) (\$?[@\w]+): (.*)/ then    # yacc
     rule = $2
     order << rule unless rules.has_key? rule
     rules[rule] << munge($3)
@@ -132,7 +204,7 @@ ARGF.each_line do |line|
   when /^\cL/ then                     # byacc
     break
   else
-    warn "unparsed: #{$.}: #{line.chomp}"
+    warn "unparsed: #{$.}: #{line.strip.inspect}"
   end
 end