ruby_parser 3.0.0 → 3.19.1

data/Manifest.txt

@@ -1,18 +1,46 @@
 .autotest
-History.txt
+History.rdoc
 Manifest.txt
-README.txt
+README.rdoc
 Rakefile
 bin/ruby_parse
 bin/ruby_parse_extract_error
-lib/gauntlet_rubyparser.rb
-lib/ruby18_parser.rb
-lib/ruby18_parser.y
-lib/ruby19_parser.rb
-lib/ruby19_parser.y
+compare/normalize.rb
+debugging.md
+gauntlet.md
+lib/.document
+lib/rp_extensions.rb
+lib/rp_stringscanner.rb
+lib/ruby20_parser.rb
+lib/ruby20_parser.y
+lib/ruby21_parser.rb
+lib/ruby21_parser.y
+lib/ruby22_parser.rb
+lib/ruby22_parser.y
+lib/ruby23_parser.rb
+lib/ruby23_parser.y
+lib/ruby24_parser.rb
+lib/ruby24_parser.y
+lib/ruby25_parser.rb
+lib/ruby25_parser.y
+lib/ruby26_parser.rb
+lib/ruby26_parser.y
+lib/ruby27_parser.rb
+lib/ruby27_parser.y
+lib/ruby30_parser.rb
+lib/ruby30_parser.y
+lib/ruby31_parser.rb
+lib/ruby31_parser.y
+lib/ruby3_parser.yy
 lib/ruby_lexer.rb
+lib/ruby_lexer.rex
+lib/ruby_lexer.rex.rb
+lib/ruby_lexer_strings.rb
 lib/ruby_parser.rb
+lib/ruby_parser.yy
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb
 test/test_ruby_parser_extras.rb
+tools/munge.rb
+tools/ripper.rb

data/{README.txt → README.rdoc}

@@ -7,25 +7,35 @@ rdoc :: http://docs.seattlerb.org/ruby_parser
 == DESCRIPTION:
 
 ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-racc--which does by default use a C extension). RP's output is
-the same as ParseTree's output: s-expressions using ruby's arrays and
-base types.
+racc--which does by default use a C extension). It outputs
+s-expressions which can be manipulated and converted back to ruby via
+the ruby2ruby gem.
 
 As an example:
 
-  def conditional1 arg1
-    return 1 if arg1 == 0
-    return 0
-  end
+    def conditional1 arg1
+      return 1 if arg1 == 0
+      return 0
+    end
 
 becomes:
 
-  s(:defn, :conditional1, s(:args, :arg1),
-    s(:if,
-      s(:call, s(:lvar, :arg1), :==, s(:lit, 0)),
-      s(:return, s(:lit, 1)),
-      nil),
-    s(:return, s(:lit, 0)))
+    s(:defn, :conditional1, s(:args, :arg1),
+      s(:if,
+        s(:call, s(:lvar, :arg1), :==, s(:lit, 0)),
+        s(:return, s(:lit, 1)),
+        nil),
+      s(:return, s(:lit, 0)))
+
+Tested against 801,039 files from the latest of all rubygems (as of 2013-05):
+
+* 1.8 parser is at 99.9739% accuracy, 3.651 sigma
+* 1.9 parser is at 99.9940% accuracy, 4.013 sigma
+* 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+* 2.6 parser is at 99.9972% accuracy, 4.191 sigma
+* 3.0 parser has a 100% parse rate.
+  * Tested against 2,672,412 unique ruby files across 167k gems.
+  * As do all the others now, basically.
 
 == FEATURES/PROBLEMS:
 
@@ -43,9 +53,29 @@ becomes:
 
 == SYNOPSIS:
 
-  Ruby19Parser.new.parse "1+1"
+  RubyParser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
+
+You can also use Ruby19Parser, Ruby18Parser, or RubyParser.for_current_ruby:
+
+  RubyParser.for_current_ruby.parse "1+1"
   # => s(:call, s(:lit, 1), :+, s(:lit, 1))
 
+== DEVELOPER NOTES:
+
+To add a new version:
+
+* New parser should be generated from lib/ruby[3]_parser.yy.
+* Extend lib/ruby[3]_parser.yy with new class name.
+* Add new version number to V2/V3 in Rakefile for rule creation.
+* Add new (full) version to `ruby_parse` section of Rakefile for rake compare
+* Require generated parser in lib/ruby_parser.rb.
+* Add empty TestRubyParserShared##Plus module and TestRubyParserV## to test/test_ruby_parser.rb.
+* Extend Manifest.txt with generated file names.
+* Add new version number to sexp_processor's pt_testcase.rb in all_versions
+
+Until all of these are done, you won't have a clean test run.
+
 == REQUIREMENTS:
 
 * ruby. woot.

data/Rakefile

@@ -1,107 +1,131 @@
 # -*- ruby -*-
 
-require 'rubygems'
-require 'hoe'
+require "rubygems"
+require "hoe"
 
 Hoe.plugin :seattlerb
 Hoe.plugin :racc
 Hoe.plugin :isolate
+Hoe.plugin :rdoc
 
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
+Hoe.add_include_dirs "../../minitest/dev/lib"
+Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
-Hoe.spec 'ruby_parser' do
-  developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
+V2   = %w[20 21 22 23 24 25 26 27]
+V3   = %w[30 31]
 
-  self.rubyforge_name = 'parsetree'
+VERS = V2 + V3
 
-  dependency 'sexp_processor', '~> 4.1'
+ENV["FAST"] = VERS.last if ENV["FAST"] && !VERS.include?(ENV["FAST"])
+VERS.replace [ENV["FAST"]] if ENV["FAST"]
 
-  if plugin? :perforce then
-    self.perforce_ignore << "lib/ruby18_parser.rb"
-    self.perforce_ignore << "lib/ruby19_parser.rb"
-  end
+Hoe.spec "ruby_parser" do
+  developer "Ryan Davis", "ryand-ruby@zenspider.com"
 
-  self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
-end
+  license "MIT"
 
-file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
-file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
+  dependency "sexp_processor", "~> 4.16"
+  dependency "rake", [">= 10", "< 15"], :developer
+  dependency "oedipus_lex", "~> 2.6", :developer
 
-task :clean do
-  rm_rf(Dir["**/*~"] +
-        Dir["**/*.diff"] +
-        Dir["coverage.info"] +
-        Dir["coverage"] +
-        Dir["lib/*.output"])
-end
+  # NOTE: Ryan!!! Stop trying to fix this dependency! Isolate just
+  # can't handle having a faux-gem half-installed! Stop! Just `gem
+  # install racc` and move on. Revisit this ONLY once racc-compiler
+  # gets split out.
 
-def next_num(glob)
-  num = Dir[glob].max[/\d+/].to_i + 1
-end
+  dependency "racc", "~> 1.5", :developer
+
+  require_ruby_version [">= 2.1", "< 4"]
+
+  if plugin? :perforce then     # generated files
+    VERS.each do |n|
+      self.perforce_ignore << "lib/ruby#{n}_parser.rb"
+    end
 
-desc "Compares PT to RP and deletes all files that match"
-task :compare do
-  files = Dir["unit/**/*.rb"]
-  puts "Parsing #{files.size} files"
-  files.each do |file|
-    puts file
-    system "./cmp.rb -q #{file} && rm #{file}"
+    VERS.each do |n|
+      self.perforce_ignore << "lib/ruby#{n}_parser.y"
+    end
+
+    self.perforce_ignore << "lib/ruby_lexer.rex.rb"
   end
-  system 'find -d unit -type d -empty -exec rmdir {} \;'
-end
 
-desc "Compares PT to RP and stops on first failure"
-task :find_bug do
-  files = Dir["unit/**/*.rb"]
-  puts "Parsing #{files.size} files"
-  files.each do |file|
-    puts file
-    sh "./cmp.rb -q #{file}"
+  if plugin?(:racc)
+    self.racc_flags << " -t" if ENV["DEBUG"]
+    self.racc_flags << " --superclass RubyParser::Parser"
+    # self.racc_flags << " --runtime ruby_parser" # TODO: broken in racc
   end
 end
 
-task :sort do
-  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
-  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
-end
+def maybe_add_to_top path, string
+  file = File.read path
 
-task :loc do
-  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
-  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
-  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
-  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+  return if file.start_with? string
 
-  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+  warn "Altering top of #{path}"
+  tmp_path = "#{path}.tmp"
+  File.open(tmp_path, "w") do |f|
+    f.puts string
+    f.puts
 
-  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
-  puts "dev  : loc = #{loc2} flog = #{flog2}"
-  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+    f.write file
+    # TODO: make this deal with encoding comments properly?
+  end
+  File.rename tmp_path, path
 end
 
-desc "Validate against all normal files in unit dir"
-task :validate do
-  sh "./cmp.rb unit/*.rb"
+def unifdef?
+  @unifdef ||= system("which unifdef") or abort <<~EOM
+    unifdef not found!
+
+    Please install 'unifdef' package on your system or `rake generate` on a mac.
+  EOM
 end
 
-def run_and_log cmd, prefix
-  files = ENV['FILES'] || 'unit/*.rb'
-  p, x = prefix, "txt"
-  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
-  f = "#{p}.#{n}.#{x}"
+V2.each do |n|
+  file "lib/ruby#{n}_parser.y" => "lib/ruby_parser.yy" do |t|
+    unifdef?
+    cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
+    sh cmd
+  end
 
-  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+  file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
+end
+
+V3.each do |n|
+  file "lib/ruby#{n}_parser.y" => "lib/ruby3_parser.yy" do |t|
+    unifdef?
+    cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
+    sh cmd
+  end
 
-  puts File.read(f)
+  file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
 end
 
-desc "Benchmark against all normal files in unit dir"
-task :benchmark do
-  run_and_log "ruby", "benchmark"
+file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
+
+task :parser do |t|
+  t.prerequisite_tasks.grep(Rake::FileTask).select(&:already_invoked).each do |f|
+    maybe_add_to_top f.name, "# frozen_string_literal: true"
+  end
+end
+
+task :generate => [:lexer, :parser]
+
+task :clean do
+  rm_rf(Dir["**/*~"] +
+        Dir["diff.diff"] + # not all diffs. bit me too many times
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby2*_parser.y"] +
+        Dir["lib/ruby3*_parser.y"] +
+        Dir["lib/*.output"])
 end
 
-desc "Profile against all normal files in unit dir"
-task :profile do
-  run_and_log "zenprofile", "profile"
+task :sort do
+  sh "grepsort '^ +def' lib/ruby_lexer.rb"
+  sh "grepsort '^ +def (test|util)' test/test_ruby_lexer.rb"
 end
 
 desc "what was that command again?"
@@ -109,80 +133,254 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
-task :irb => [:isolate] do
-  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
-end
-
 def (task(:phony)).timestamp
   Time.at 0
 end
 
 task :isolate => :phony
 
-file "lib/ruby18_parser.rb" => :isolate
-file "lib/ruby19_parser.rb" => :isolate
+def in_compare
+  Dir.chdir "compare" do
+    yield
+  end
+end
 
-task :compare18 do
-  sh "./yack.rb lib/ruby18_parser.output > racc18.txt"
-  sh "./yack.rb parse18.output > yacc18.txt"
-  sh "diff -du racc18.txt yacc18.txt || true"
-  puts
-  sh "diff -du racc18.txt yacc18.txt | wc -l"
+def dl v
+  dir = v[/^\d+\.\d+/]
+  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.xz"
+  path = File.basename url
+  unless File.exist? path then
+    system "curl -O #{url}"
+  end
 end
 
-task :compare19 do
-  sh "./yack.rb lib/ruby19_parser.output > racc19.txt"
-  sh "./yack.rb parse19.output > yacc19.txt"
-  sh "diff -du racc19.txt yacc19.txt || true"
-  puts
-  sh "diff -du racc19.txt yacc19.txt | wc -l"
+def ruby_parse version
+  v         = version[/^\d+\.\d+/].delete "."
+  rp_txt    = "rp#{v}.txt"
+  mri_txt   = "mri#{v}.txt"
+  parse_y   = "parse#{v}.y"
+  tarball   = "ruby-#{version}.tar.xz"
+  ruby_dir  = "ruby-#{version}"
+  diff      = "diff#{v}.diff"
+  rp_out    = "lib/ruby#{v}_parser.output"
+  _rp_y     = "lib/ruby#{v}_parser.y"
+  rp_y_rb   = "lib/ruby#{v}_parser.rb"
+
+  c_diff    = "compare/#{diff}"
+  c_rp_txt  = "compare/#{rp_txt}"
+  c_mri_txt = "compare/#{mri_txt}"
+  c_parse_y = "compare/#{parse_y}"
+  c_tarball = "compare/#{tarball}"
+  normalize = "compare/normalize.rb"
+
+  file c_tarball do
+    in_compare do
+      dl version
+    end
+  end
+
+  desc "fetch all tarballs"
+  task :fetch => c_tarball
+
+  file c_parse_y => c_tarball do
+    in_compare do
+      extract_glob = case
+                     when version > "2.7" then
+                       "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                     else
+                       "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                     end
+      system "tar Jxf #{tarball} #{ruby_dir}/#{extract_glob}"
+
+      Dir.chdir ruby_dir do
+        if File.exist? "tool/id2token.rb" then
+          sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
+        else
+          sh "expand parse.y > ../#{parse_y}"
+        end
+
+        ruby "-pi", "-e", 'gsub(/^%pure-parser/, "%define api.pure")', "../#{parse_y}"
+      end
+      sh "rm -rf #{ruby_dir}"
+    end
+  end
+
+  bison = Dir["/opt/homebrew/opt/bison/bin/bison",
+              "/usr/local/opt/bison/bin/bison",
+              `which bison`.chomp,
+             ].first
+
+  file c_mri_txt => [c_parse_y, normalize] do
+    in_compare do
+      sh "#{bison} -r all #{parse_y}"
+      sh "./normalize.rb parse#{v}.output > #{mri_txt}"
+      rm ["parse#{v}.output", "parse#{v}.tab.c"]
+    end
+  end
+
+  file rp_out => rp_y_rb
+
+  file c_rp_txt => [rp_out, normalize] do
+    in_compare do
+      sh "./normalize.rb ../#{rp_out} > #{rp_txt}"
+    end
+  end
+
+  compare = "compare#{v}"
+
+  desc "Compare all grammars to MRI"
+  task :compare => compare
+
+  file c_diff => [c_mri_txt, c_rp_txt] do
+    in_compare do
+      sh "diff -du #{mri_txt} #{rp_txt} > #{diff}; true"
+    end
+  end
+
+  desc "Compare #{v} grammar to MRI #{version}"
+  task compare => c_diff do
+    in_compare do
+      system "wc -l #{diff}"
+    end
+  end
+
+  task :clean do
+    rm_f Dir[c_mri_txt, c_rp_txt]
+  end
+
+  task :realclean do
+    rm_f Dir[c_parse_y, tarball]
+  end
+end
+
+task :versions do
+  require "open-uri"
+  require "net/http" # avoid require issues in threads
+  require "net/https"
+
+  versions = VERS.map { |s| s.split(//).join "." }
+
+  base_url = "https://cache.ruby-lang.org/pub/ruby"
+
+  class Array
+    def human_sort
+      sort_by { |item| item.to_s.split(/(\d+)/).map { |e| [e.to_i, e] } }
+    end
+  end
+
+  versions = versions.map { |ver|
+    Thread.new {
+      URI
+        .parse("#{base_url}/#{ver}/")
+        .read
+        .scan(/ruby-\d+\.\d+\.\d+[-\w.]*?.tar.gz/)
+        .reject { |s| s =~ /-(?:rc|preview)\d/ }
+        .human_sort
+        .last
+        .delete_prefix("ruby-")
+        .delete_suffix ".tar.gz"
+    }
+  }.map(&:value).sort
+
+  puts versions.map { |v| "ruby_parse %p" % [v] }
 end
 
+ruby_parse "2.0.0-p648"
+ruby_parse "2.1.10"
+ruby_parse "2.2.10"
+ruby_parse "2.3.8"
+ruby_parse "2.4.10"
+ruby_parse "2.5.9"
+ruby_parse "2.6.9"
+ruby_parse "2.7.5"
+ruby_parse "3.0.3"
+ruby_parse "3.1.1"
+
 task :debug => :isolate do
-  ENV["V"] ||= "19"
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
+  Rake.application[:lexer].invoke # this way we can have DEBUG set
+
+  $:.unshift "lib"
+  require "ruby_parser"
+  require "pp"
 
-  $: << "lib"
-  require 'ruby_parser'
+  klass = Object.const_get("Ruby#{ENV["V"]}Parser") rescue nil
+  raise "Unsupported version #{ENV["V"]}" unless klass
+  parser = klass.new
 
-  parser = if ENV["V"] == "18" then
-             Ruby18Parser.new
-           else
-             Ruby19Parser.new
-           end
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
 
-  file = ENV["F"] || ENV["FILE"]
+  n = ENV["BUG"]
+  file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "debug.rb"
+  ruby = ENV["R"] || ENV["RUBY"]
+
+  if ruby then
+    file = "env"
+  else
+    ruby = File.read file
+  end
 
-  ruby = if file then
-           File.read(file)
-         else
-           file = "env"
-           ENV["R"] || ENV["RUBY"]
-         end
 
   begin
-    p parser.process(ruby, file)
-  rescue Racc::ParseError => e
+    pp parser.process(ruby, file, time)
+  rescue ArgumentError, Racc::ParseError => e
     p e
-    ss = parser.lexer.src
+    puts e.backtrace.join "\n  "
+    ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
     abort "on #{file}:#{lines.size}"
   end
 end
 
-task :debug_ruby do
-  file = ENV["F"] || ENV["FILE"]
-  sh "ruby19 -cwy #{file} 2>&1 | ./yuck.rb"
+task :debug3 do
+  file    = ENV["F"] || "debug.rb"
+  version = ENV["V"] || ""
+  verbose = ENV["VERBOSE"] ? "-v" : ""
+  munge    = "./tools/munge.rb #{verbose}"
+
+  abort "Need a file to parse, via: F=path.rb" unless file
+
+  ENV.delete "V"
+
+  ruby = "ruby#{version}"
+
+  sh "#{ruby} -v"
+  sh "#{ruby} -y #{file} 2>&1 | #{munge} > tmp/ruby"
+  sh "#{ruby} ./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
+  sh "diff -U 999 -d tmp/{ruby,rp}"
+end
+
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
+end
+
+task :cmp3 do
+  sh %(emacsclient --eval '(ediff-files3 "tmp/ruby" "tmp/rip" "tmp/rp")')
 end
 
 task :extract => :isolate do
-  ENV["V"] ||= "19"
+  ENV["V"] ||= VERS.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
-  file = ENV["F"] || ENV["FILE"]
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
 
   ruby "-Ilib", "bin/ruby_parse_extract_error", file
 end
 
+task :parse => :isolate do
+  ENV["V"] ||= VERS.last
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
+
+  ruby "-Ilib", "bin/ruby_parse", file
+end
+
+task :bugs do
+  sh "for f in bug*.rb bad*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
+end
+
 # vim: syntax=Ruby

data/bin/ruby_parse

@@ -26,6 +26,8 @@ class File
 end
 
 begin
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   ARGV.each do |file|
     rp = RubyParser.new
     loc = `wc -l #{file}`.strip.to_i
@@ -45,7 +47,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.binread(file), file)
+        r = rp.process(File.binread(file), file, time)
         pp r unless $q
         good += 1
       rescue SyntaxError => e