ruby_parser 3.13.1 → 3.21.0

data/Rakefile

@@ -1,6 +1,5 @@
 # -*- ruby -*-
 
-require "rubygems"
 require "hoe"
 
 Hoe.plugin :seattlerb
@@ -8,29 +7,41 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 Hoe.plugin :rdoc
 
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
+Hoe.add_include_dirs "../../ruby2ruby/dev/lib"
 
-V2   = %w[20 21 22 23 24 25 26]
-V2.replace [V2.last] if ENV["FAST"] # HACK
+V2   = %w[20 21 22 23 24 25 26 27]
+V3   = %w[30 31 32 33]
+
+VERS = V2 + V3
+
+ENV["FAST"] = VERS.last if ENV["FAST"] && !VERS.include?(ENV["FAST"])
+VERS.replace [ENV["FAST"]] if ENV["FAST"]
+
+racc_flags = nil
 
 Hoe.spec "ruby_parser" do
   developer "Ryan Davis", "ryand-ruby@zenspider.com"
 
   license "MIT"
 
-  dependency "sexp_processor", "~> 4.9"
-  dependency "rake", "< 11", :developer
-  dependency "oedipus_lex", "~> 2.5", :developer
+  dependency "sexp_processor", "~>  4.16"
+  dependency "racc",           "~>  1.5"
+  dependency "rake",           [">= 10",  "< 15"], :developer
+  dependency "oedipus_lex",    "~>  2.6", :developer
+
+  require_ruby_version [">= 2.6", "< 4"]
 
   if plugin? :perforce then     # generated files
-    V2.each do |n|
-      self.perforce_ignore << "lib/ruby#{n}_parser.rb"
+    VERS.each do |n|
+      self.perforce_ignore << "lib/ruby_parser#{n}.rb"
     end
 
-    V2.each do |n|
-      self.perforce_ignore << "lib/ruby#{n}_parser.y"
+    VERS.each do |n|
+      self.perforce_ignore << "lib/ruby_parser#{n}.y"
     end
 
     self.perforce_ignore << "lib/ruby_lexer.rex.rb"
@@ -39,27 +50,68 @@ Hoe.spec "ruby_parser" do
   if plugin?(:racc)
     self.racc_flags << " -t" if ENV["DEBUG"]
     self.racc_flags << " --superclass RubyParser::Parser"
-    # self.racc_flags << " --runtime ruby_parser" # TODO: broken in racc
+    racc_flags = self.racc_flags
   end
 end
 
-V2.each do |n|
-  file "lib/ruby#{n}_parser.y" => "lib/ruby_parser.yy" do |t|
-    cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
-    sh cmd
+def maybe_add_to_top path, string
+  file = File.read path
+
+  return if file.start_with? string
+
+  warn "Altering top of #{path}"
+  tmp_path = "#{path}.tmp"
+  File.open(tmp_path, "w") do |f|
+    f.puts string
+    f.puts
+
+    f.write file
+    # TODO: make this deal with encoding comments properly?
   end
+  File.rename tmp_path, path
+end
+
+def unifdef?
+  @unifdef ||= system("which unifdef") or abort <<~EOM
+    unifdef not found!
+
+    Please install 'unifdef' package on your system or `rake generate` on a mac.
+  EOM
+end
+
+def racc?
+  @racc ||= system("which racc") or abort <<~EOM
+    racc not found! `gem install racc`
+  EOM
+end
 
-  file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
+generate_parser = proc do |t|
+  unifdef?
+  racc?
+  n = t.name[/\d+/]
+  sh "unifdef -tk -DV=%s %s | racc %s /dev/stdin -o %s" % [n, t.source, racc_flags, t.name]
+  maybe_add_to_top t.name, "# frozen_string_literal: true"
+end
+
+V2.each do |n|
+  file "lib/ruby_parser#{n}.rb" => "lib/ruby_parser2.yy", &generate_parser
+end
+
+V3.each do |n|
+  file "lib/ruby_parser#{n}.rb" => "lib/ruby_parser3.yy", &generate_parser
 end
 
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
+task :generate => [:lexer, :parser]
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["diff.diff"] + # not all diffs. bit me too many times
         Dir["coverage.info"] +
         Dir["coverage"] +
-        Dir["lib/ruby2*_parser.y"] +
+        Dir["lib/ruby_parser2*.y"] +
+        Dir["lib/ruby_parser3*.y"] +
         Dir["lib/*.output"])
 end
 
@@ -73,127 +125,187 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
-task :irb => [:isolate] do
-  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
-end
-
 def (task(:phony)).timestamp
   Time.at 0
 end
 
 task :isolate => :phony
 
-def in_compare
-  Dir.chdir "compare" do
-    yield
-  end
-end
-
-def dl v
+def dl v, f
   dir = v[/^\d+\.\d+/]
-  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.bz2"
-  path = File.basename url
-  unless File.exist? path then
-    system "curl -O #{url}"
-  end
+  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.xz"
+
+  warn "Downloading ruby #{v}"
+  system "curl -s -o #{f} #{url}"
 end
 
+task :parser => :isolate
+
+multitask :compare_build
+task :compare_build => :generate
+task :compare => :compare_build
+
 def ruby_parse version
   v         = version[/^\d+\.\d+/].delete "."
-  rp_txt    = "rp#{v}.txt"
-  mri_txt   = "mri#{v}.txt"
-  parse_y   = "parse#{v}.y"
-  tarball   = "ruby-#{version}.tar.bz2"
-  ruby_dir  = "ruby-#{version}"
-  diff      = "diff#{v}.diff"
-  rp_out    = "lib/ruby#{v}_parser.output"
-  rp_y      = "lib/ruby#{v}_parser.y"
-  rp_y_rb   = "lib/ruby#{v}_parser.rb"
-
-  c_diff    = "compare/#{diff}"
-  c_rp_txt  = "compare/#{rp_txt}"
-  c_mri_txt = "compare/#{mri_txt}"
-  c_parse_y = "compare/#{parse_y}"
-  c_tarball = "compare/#{tarball}"
+  diff      = "compare/diff#{v}.diff"
+  rp_txt    = "compare/rp#{v}.txt"
+  mri_txt   = "compare/mri#{v}.txt"
+  parse_y   = "compare/parse#{v}.y"
+  tarball   = "compare/ruby-#{version}.tar.xz"
+  ruby_dir  = "compare/ruby-#{version}"
+  rp_out    = "lib/ruby_parser#{v}.output"
+  rp_y_rb   = "lib/ruby_parser#{v}.rb"
   normalize = "compare/normalize.rb"
 
-  file c_tarball do
-    in_compare do
-      dl version
-    end
+  file tarball do
+    dl version, tarball
   end
 
-  file c_parse_y => c_tarball do
-    in_compare do
-      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
-      Dir.chdir ruby_dir do
-        if File.exist? "tool/id2token.rb" then
-          sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
-        else
-          sh "expand parse.y > ../#{parse_y}"
-        end
-      end
-      sh "rm -rf #{ruby_dir}"
+  desc "fetch all tarballs"
+  task :fetch => tarball
+
+  file parse_y => tarball do
+    extract_glob = case
+                   when version > "3.3" then
+                     "{id.h,parse.y,tool/{id2token.rb,lrama},defs/id.def}"
+                   when version > "3.2" then
+                     "{id.h,parse.y,tool/id2token.rb,defs/id.def}"
+                   when version > "2.7" then
+                     "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                   else
+                     "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                   end
+    system "tar xf #{tarball} -C compare #{File.basename ruby_dir}/#{extract_glob}"
+
+    # Debugging a new parse build system:
+    #
+    # Unpack the ruby tarball in question, configure, and run the following:
+    #
+    # % touch parse.y; make -n parse.c
+    # ...
+    # echo generating parse.c
+    # /Users/ryan/.rubies.current/bin/ruby --disable=gems ./tool/id2token.rb parse.y | \
+    #       ruby ./tool/lrama/exe/lrama -oparse.c -Hparse.h - parse.y
+    #
+    # Then integrate these commands into the mess below:
+
+    d = ruby_dir
+    cmd = if version > "3.2" then
+            "ruby #{d}/tool/id2token.rb #{d}/parse.y | expand > #{parse_y}"
+          else
+            "ruby #{d}/tool/id2token.rb --path-separator=.:./ #{d}/id.h #{d}/parse.y | expand | ruby -pe 'gsub(/^%pure-parser/, \"%define api.pure\")'  > #{parse_y}"
+          end
+
+    sh cmd
+
+    if File.exist? "#{d}/tool/lrama" then # UGH: this is dumb
+      rm_rf "compare/lrama"
+      sh "mv #{d}/tool/lrama compare"
     end
+    sh "rm -rf #{d}"
   end
 
-  file c_mri_txt => [c_parse_y, normalize] do
-    in_compare do
-      sh "bison -r all #{parse_y}"
-      sh "./normalize.rb parse#{v}.output > #{mri_txt}"
-      rm ["parse#{v}.output", "parse#{v}.tab.c"]
+  bison = Dir["/opt/homebrew/opt/bison/bin/bison",
+              "/usr/local/opt/bison/bin/bison",
+              `which bison`.chomp,
+             ].first
+
+  file mri_txt => [parse_y, normalize] do
+    if version > "3.3" then
+      sh "./compare/lrama/exe/lrama -r all -ocompare/parse#{v}.tab.c #{parse_y}"
+    else
+      sh "#{bison} -r all #{parse_y}"
+      mv Dir["parse#{v}.*"], "compare"
     end
+
+    sh "#{normalize} compare/parse#{v}.output > #{mri_txt}"
+    rm ["compare/parse#{v}.output", "compare/parse#{v}.tab.c"]
   end
 
   file rp_out => rp_y_rb
 
-  file c_rp_txt => [rp_out, normalize] do
-    in_compare do
-      sh "./normalize.rb ../#{rp_out} > #{rp_txt}"
-    end
+  file rp_txt => [rp_out, normalize] do
+    sh "#{normalize} #{rp_out} > #{rp_txt}"
   end
 
   compare = "compare#{v}"
+  compare_build = "compare_build#{v}"
 
   desc "Compare all grammars to MRI"
   task :compare => compare
+  task :compare_build => compare_build
 
-  file c_diff => [c_mri_txt, c_rp_txt] do
-    in_compare do
-      sh "diff -du #{mri_txt} #{rp_txt} > #{diff}; true"
-    end
+  task compare_build => diff
+
+  file diff => [mri_txt, rp_txt] do
+    sh "diff -du #{mri_txt} #{rp_txt} > #{diff}; true"
   end
 
   desc "Compare #{v} grammar to MRI #{version}"
-  task compare => c_diff do
-    in_compare do
-      system "wc -l #{diff}"
-    end
+  task compare => diff do
+    system "wc -l #{diff}"
   end
 
   task :clean do
-    rm_f Dir[c_mri_txt, c_rp_txt]
+    rm_f Dir[mri_txt, rp_txt]
   end
 
   task :realclean do
-    rm_f Dir[c_parse_y, tarball]
+    rm_f Dir[parse_y, tarball]
+  end
+end
+
+task :versions do
+  require "open-uri"
+  require "net/http" # avoid require issues in threads
+  require "net/https"
+
+  versions = VERS.map { |s| s.split(//).join "." }
+
+  base_url = "https://cache.ruby-lang.org/pub/ruby"
+
+  class Array
+    def human_sort
+      sort_by { |item| item.to_s.split(/(\d+)/).map { |e| [e.to_i, e] } }
+    end
   end
+
+  versions = versions.map { |ver|
+    Thread.new {
+      URI
+        .parse("#{base_url}/#{ver}/")
+        .read
+        .scan(/ruby-\d+\.\d+\.\d+[-\w.]*?.tar.gz/)
+        .reject { |s| s =~ /-(?:rc|preview)\d/ }
+        .human_sort
+        .last
+        .delete_prefix("ruby-")
+        .delete_suffix ".tar.gz"
+    }
+  }.map(&:value).sort
+
+  puts versions.map { |v| "ruby_parse %p" % [v] }
 end
 
 ruby_parse "2.0.0-p648"
-ruby_parse "2.1.9"
-ruby_parse "2.2.9"
+ruby_parse "2.1.10"
+ruby_parse "2.2.10"
 ruby_parse "2.3.8"
-ruby_parse "2.4.5"
-ruby_parse "2.5.3"
-ruby_parse "2.6.1"
+ruby_parse "2.4.10"
+ruby_parse "2.5.9"
+ruby_parse "2.6.10"
+ruby_parse "2.7.8"
+ruby_parse "3.0.6"
+ruby_parse "3.1.4"
+ruby_parse "3.2.2"
+ruby_parse "3.3.0"
 
 task :debug => :isolate do
-  ENV["V"] ||= V2.last
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
-  $: << "lib"
+  $:.unshift "lib"
   require "ruby_parser"
   require "pp"
 
@@ -204,19 +316,21 @@ task :debug => :isolate do
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   n = ENV["BUG"]
-  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"]
+  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "debug.rb"
+  ruby = ENV["R"] || ENV["RUBY"]
+
+  if ruby then
+    file = "env"
+  else
+    ruby = File.read file
+  end
 
-  ruby = if file then
-           File.read(file)
-         else
-           file = "env"
-           ENV["R"] || ENV["RUBY"]
-         end
 
   begin
     pp parser.process(ruby, file, time)
-  rescue Racc::ParseError => e
+  rescue ArgumentError, Racc::ParseError => e
     p e
+    puts e.backtrace.join "\n  "
     ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
@@ -225,18 +339,26 @@ task :debug => :isolate do
 end
 
 task :debug3 do
-  file    = ENV["F"]
-  verbose = ENV["V"] ? "-v" : ""
+  file    = ENV["F"] || "debug.rb"
+  version = ENV["V"] || ""
+  verbose = ENV["VERBOSE"] ? "-v" : ""
   munge    = "./tools/munge.rb #{verbose}"
 
   abort "Need a file to parse, via: F=path.rb" unless file
 
   ENV.delete "V"
 
-  sh "ruby -y #{file} 2>&1 | #{munge} > tmp/ruby"
-  sh "./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
-  sh "rake debug F=#{file} DEBUG=1 V=25 2>&1 | #{munge} > tmp/rp"
-  sh "diff -U 999 -d tmp/{rip,rp}"
+  ruby = "ruby#{version}"
+
+  sh "#{ruby} -v"
+  sh "#{ruby} -y #{file} 2>&1 | #{munge} > tmp/ruby"
+  sh "#{ruby} ./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
+  sh "diff -U 999 -d tmp/{ruby,rp}"
+end
+
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
 end
 
 task :cmp3 do
@@ -244,16 +366,25 @@ task :cmp3 do
 end
 
 task :extract => :isolate do
-  ENV["V"] ||= V2.last
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
-  file = ENV["F"] || ENV["FILE"]
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
 
   ruby "-Ilib", "bin/ruby_parse_extract_error", file
 end
 
+task :parse => :isolate do
+  ENV["V"] ||= VERS.last
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
+
+  ruby "-Ilib", "bin/ruby_parse", file
+end
+
 task :bugs do
-  sh "for f in bug*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
+  sh "for f in bug*.rb bad*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
 end
 
 # vim: syntax=Ruby

data/bin/ruby_parse

@@ -26,6 +26,8 @@ class File
 end
 
 begin
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   ARGV.each do |file|
     rp = RubyParser.new
     loc = `wc -l #{file}`.strip.to_i
@@ -45,7 +47,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.binread(file), file)
+        r = rp.process(File.binread(file), file, time)
         pp r unless $q
         good += 1
       rescue SyntaxError => e

data/bin/ruby_parse_extract_error

@@ -21,7 +21,7 @@ class RubyParser
     src = ss.string
     pre_error = src[0...ss.pos]
 
-    defs = pre_error.grep(/^ *(?:def|it)/)
+    defs = pre_error.lines.grep(/^ *(?:def|it)/)
 
     raise "can't figure out where the bad code starts" unless defs.last
 
@@ -104,9 +104,14 @@ rescue Timeout::Error
   warn "TIMEOUT parsing #{file}. Skipping."
 
   if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
   elsif $t then
     File.unlink file
   end

data/compare/normalize.rb

@@ -8,6 +8,10 @@ order = []
 
 def munge s
   renames = [
+             # unquote... wtf?
+             /`(.+?)'/,          proc { $1 },
+             /"'(.+?)'"/,        proc { "\"#{$1}\"" },
+
              "'='",             "tEQL",
              "'!'",             "tBANG",
              "'%'",             "tPERCENT",
@@ -80,6 +84,8 @@ def munge s
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
+             "ε",               "none", # bison 3+
+             "%empty",          "none", # newer bison
              "/* empty */",     "none",
              /^\s*$/,           "none",
 
@@ -100,6 +106,44 @@ def munge s
 
              "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
 
+             # 2.7 changes:
+
+             '"global variable"',          "tGVAR",
+             '"operator-assignment"',      "tOP_ASGN",
+             '"back reference"',           "tBACK_REF",
+             '"numbered reference"',       "tNTH_REF",
+             '"local variable or method"', "tIDENTIFIER",
+             '"constant"',                 "tCONSTANT",
+
+             '"(.."',                  "tBDOT2",
+             '"(..."',                 "tBDOT3",
+             '"char literal"',         "tCHAR",
+             '"literal content"',      "tSTRING_CONTENT",
+             '"string literal"',       "tSTRING_BEG",
+             '"symbol literal"',       "tSYMBEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+
+             '"float literal"',        "tFLOAT",
+             '"imaginary literal"',    "tIMAGINARY",
+             '"integer literal"',      "tINTEGER",
+             '"rational literal"',     "tRATIONAL",
+
+             '"instance variable"',  "tIVAR",
+             '"class variable"',     "tCVAR",
+             '"terminator"',         "tSTRING_END", # TODO: switch this?
+             '"method"',             "tFID",
+             '"}"',                  "tSTRING_DEND",
+
+             '"do for block"',     "kDO_BLOCK",
+             '"do for condition"', "kDO_COND",
+             '"do for lambda"',    "kDO_LAMBDA",
+             "tLABEL",             "kLABEL",
+
              # UGH
              "k_LINE__",       "k__LINE__",
              "k_FILE__",       "k__FILE__",
@@ -107,15 +151,17 @@ def munge s
 
              '"defined?"',     "kDEFINED",
 
-
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
 
-             /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
-             /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
 
-             /@(\d+)(\s+|$)/,       "",
+             /\$?@(\d+)(\s+|$)/,    "", # newer bison
+
+             # TODO: remove for 3.0 work:
+             "lex_ctxt ", "" # 3.0 production that's mostly noise right now
             ]
 
   renames.each_slice(2) do |(a, b)|
@@ -134,7 +180,7 @@ ARGF.each_line do |line|
 
   case line.strip
   when /^$/ then
-  when /^(\d+) (\$?\w+): (.*)/ then    # yacc
+  when /^(\d+) (\$?[@\w]+): (.*)/ then    # yacc
     rule = $2
     order << rule unless rules.has_key? rule
     rules[rule] << munge($3)
@@ -156,10 +202,12 @@ ARGF.each_line do |line|
     break
   when /^Terminals/ then               # yacc
     break
+  when /^State \d/ then                # lrama
+    break
   when /^\cL/ then                     # byacc
     break
   else
-    warn "unparsed: #{$.}: #{line.chomp}"
+    warn "unparsed: #{$.}: #{line.strip.inspect}"
   end
 end