ruby_parser 3.8.4 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0abf3a43140d5d77875d4ae77c8657ff6e011b9
-  data.tar.gz: f7634c27ce13a769c186cda5763d9597d68f1202
+  metadata.gz: db3aec0f342263aec6c4bc598f2af937d889219b
+  data.tar.gz: 0a7300cfcb5109443168aa3432eefb0b103a07a1
 SHA512:
-  metadata.gz: 4dca307209ff4d2d19ce5aeebe96a4368abb74acd38837988b4950c21eedc7a0bf48cfdf3155547b585050e97e1a2ab84baee6abf12b54247084e74ae3d40d7b
-  data.tar.gz: 3ce888e70acce944e24c30883600314d34cc95ff32a6df6ce7ae32c8aa01d7dfea9e93e42c27c3e9954dc6288248f82b8c5fe2f978e0adaf955df3042299668a
+  metadata.gz: c26c726384052dc9ea43d9cb29f982ed03768458c1ff0e64a515026c721be346f009ead616d18c499d449fbf9a6c23234a4c637afb14cf28fb74035a33a743db
+  data.tar.gz: 4f3d570afe84856c5275be5a9eaf9277168a89db5bd3dea605cc86cf8c38492d9ee0b5ec2d32c3413124c068ff57e24a5241d8474463e15018376328fff024ec

data/History.rdoc

@@ -1,3 +1,32 @@
+=== 3.9.0 / 2017-04-13
+
+* 1 major enhancement:
+
+  * Added tentative ruby 2.4 support. Probably missing things.
+
+* 13 minor enhancements:
+
+  * Added RubyParser.latest.
+  * Added RubyParser::Parser.version to make version range comparisons easier
+  * Changed superclasses of all parsers to RubyParser::Parser
+  * Cleaned up a lot of the version specific code to be more generic/flexible.
+  * Documented how to add new versions in README.
+  * Moved RubyParser from ruby_parser_extras.rb into ruby_parser.rb to fix bootstrap issues.
+  * Renamed RubyParserStuff#get_match_node to new_match. Aliased and deprecated.
+  * Renamed RubyParserStuff#logop to logical_op. Aliased and deprecated.
+  * Renamed RubyParserStuff#node_assign to new_assign. Aliased and deprecated.
+  * Renamed all parsers to RubyParser::V##.
+  * Revamped grammar preprocessing to make adding new versions easier.
+  * RubyParser.for_current_ruby falls back to latest if current not available.
+  * Subclasses of RubyParser::Parser register themselves into RubyParser::VERSIONS.
+
+* 4 bug fixes:
+
+  * Fixed `&.` after newline. (presidentbeef)
+  * Fixed bug setting line number for hash literals to line of opening brace.
+  * Fixed grammar preprocessing bug.
+  * Properly handle kDO with no-parens stabby lambda. (presidentbeef)
+
 === 3.8.4 / 2017-01-13
 
 * 3 bug fixes:

data/Manifest.txt

@@ -5,7 +5,10 @@ README.rdoc
 Rakefile
 bin/ruby_parse
 bin/ruby_parse_extract_error
+compare/normalize.rb
 lib/.document
+lib/rp_extensions.rb
+lib/rp_stringscanner.rb
 lib/ruby18_parser.rb
 lib/ruby18_parser.y
 lib/ruby19_parser.rb
@@ -18,6 +21,8 @@ lib/ruby22_parser.rb
 lib/ruby22_parser.y
 lib/ruby23_parser.rb
 lib/ruby23_parser.y
+lib/ruby24_parser.rb
+lib/ruby24_parser.y
 lib/ruby_lexer.rb
 lib/ruby_lexer.rex
 lib/ruby_lexer.rex.rb

data/README.rdoc

@@ -57,6 +57,18 @@ You can also use Ruby19Parser, Ruby18Parser, or RubyParser.for_current_ruby:
   RubyParser.for_current_ruby.parse "1+1"
   # => s(:call, s(:lit, 1), :+, s(:lit, 1))
 
+== DEVELOPER NOTES:
+
+To add a new version:
+
+* New parser should be generated from lib/ruby_parser.yy.
+* Extend lib/ruby_parser.yy with new class name.
+* Add new version number to Rakefile for rule creation.
+* Require generated parser in lib/ruby_parser.rb.
+* Add empty TestRubyParserShared##Plus module and TestRubyParserV## to test/test_ruby_parser.rb.
+* Extend Manifest.txt with generated file names.
+* Extend sexp_processor's pt_testcase.rb to match version
+
 == REQUIREMENTS:
 
 * ruby. woot.

data/Rakefile

@@ -14,6 +14,10 @@ Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
+V1   = %w[18 19]
+V2   = %w[20 21 22 23 24]
+V1_2 = V1 + V2
+
 Hoe.spec "ruby_parser" do
   developer "Ryan Davis", "ryand-ruby@zenspider.com"
 
@@ -24,45 +28,35 @@ Hoe.spec "ruby_parser" do
   dependency "oedipus_lex", "~> 2.1", :developer
 
   if plugin? :perforce then     # generated files
-    self.perforce_ignore << "lib/ruby18_parser.rb"
-    self.perforce_ignore << "lib/ruby19_parser.rb"
-    self.perforce_ignore << "lib/ruby20_parser.rb"
-    self.perforce_ignore << "lib/ruby20_parser.y"
-    self.perforce_ignore << "lib/ruby21_parser.rb"
-    self.perforce_ignore << "lib/ruby21_parser.y"
-    self.perforce_ignore << "lib/ruby22_parser.rb"
-    self.perforce_ignore << "lib/ruby22_parser.y"
-    self.perforce_ignore << "lib/ruby23_parser.rb"
-    self.perforce_ignore << "lib/ruby23_parser.y"
-    self.perforce_ignore << "lib/ruby_lexer.rex.rb"
-  end
+    V1_2.each do |n|
+      self.perforce_ignore << "lib/ruby#{n}_parser.rb"
+    end
 
-  self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
-end
+    V2.each do |n|
+      self.perforce_ignore << "lib/ruby#{n}_parser.y"
+    end
 
-file "lib/ruby20_parser.y" => "lib/ruby_parser.yy" do |t|
-  sh "unifdef -tk -DRUBY20 -URUBY21 -URUBY22 -URUBY23 -UDEAD #{t.source} > #{t.name} || true"
-end
+    self.perforce_ignore << "lib/ruby_lexer.rex.rb"
+  end
 
-file "lib/ruby21_parser.y" => "lib/ruby_parser.yy" do |t|
-  sh "unifdef -tk -URUBY20 -DRUBY21 -URUBY22 -URUBY23 -UDEAD #{t.source} > #{t.name} || true"
+  if plugin?(:racc)
+    self.racc_flags << " -t" if ENV["DEBUG"]
+    self.racc_flags << " --superclass RubyParser::Parser"
+    # self.racc_flags << " --runtime ruby_parser" # TODO: broken in racc
+  end
 end
 
-file "lib/ruby22_parser.y" => "lib/ruby_parser.yy" do |t|
-  sh "unifdef -tk -URUBY20 -URUBY21 -DRUBY22 -URUBY23 -UDEAD #{t.source} > #{t.name} || true"
+V2.each do |n|
+  file "lib/ruby#{n}_parser.y" => "lib/ruby_parser.yy" do |t|
+    cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
+    sh cmd
+  end
 end
 
-file "lib/ruby23_parser.y" => "lib/ruby_parser.yy" do |t|
-  sh "unifdef -tk -URUBY20 -URUBY21 -URUBY22 -DRUBY23 -UDEAD #{t.source} > #{t.name} || true"
+V1_2.each do |n|
+  file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
 end
 
-
-file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
-file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
-file "lib/ruby20_parser.rb" => "lib/ruby20_parser.y"
-file "lib/ruby21_parser.rb" => "lib/ruby21_parser.y"
-file "lib/ruby22_parser.rb" => "lib/ruby22_parser.y"
-file "lib/ruby23_parser.rb" => "lib/ruby23_parser.y"
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
 task :clean do
@@ -94,32 +88,110 @@ end
 
 task :isolate => :phony
 
-# to create parseXX.output:
-#
-# 1) check out the XX version of ruby
-# 2) Edit uncommon.mk, find the ".y.c" rule and remove the RM lines
-# 3) run `rm -f parse.c; make parse.c`
-# 4) run `bison -r all parse.tmp.y`
-# 5) mv parse.tmp.output parseXX.output
-
-# possibly new instructions:
-#
-# 1) check out the XX version of ruby
-# 2) YFLAGS="-r all" make parse.c
-# 3) mv y.output parseXX.output
-
-%w[18 19 20 21 22 23].each do |v|
-  task "compare#{v}" do
-    sh "./yack.rb lib/ruby#{v}_parser.output > racc#{v}.txt"
-    sh "./yack.rb parse#{v}.output > yacc#{v}.txt"
-    sh "diff -du racc#{v}.txt yacc#{v}.txt || true"
-    puts
-    sh "diff -du racc#{v}.txt yacc#{v}.txt | wc -l"
+def in_compare
+  Dir.chdir "compare" do
+    yield
+  end
+end
+
+def dl v
+  dir = v[/^\d+\.\d+/]
+  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.bz2"
+  path = File.basename url
+  unless File.exist? path then
+    system "curl -O #{url}"
+  end
+end
+
+def ruby_parse version
+  v         = version[/^\d+\.\d+/].delete "."
+  rp_txt    = "rp#{v}.txt"
+  mri_txt   = "mri#{v}.txt"
+  parse_y   = "parse#{v}.y"
+  tarball   = "ruby-#{version}.tar.bz2"
+  ruby_dir  = "ruby-#{version}"
+  diff      = "diff#{v}.diff"
+  rp_out    = "lib/ruby#{v}_parser.output"
+
+  c_diff    = "compare/#{diff}"
+  c_rp_txt  = "compare/#{rp_txt}"
+  c_mri_txt = "compare/#{mri_txt}"
+  c_parse_y = "compare/#{parse_y}"
+  c_tarball = "compare/#{tarball}"
+
+  file tarball do
+    in_compare do
+      dl version
+    end
+  end
+
+  file c_parse_y => c_tarball do
+    in_compare do
+      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+      Dir.chdir ruby_dir do
+        if File.exist? "tool/id2token.rb" then
+          sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y > ../#{parse_y}"
+        else
+          cp "parse.y", "../#{parse_y}"
+        end
+      end
+      sh "rm -rf #{ruby_dir}"
+    end
+  end
+
+  file c_mri_txt => c_parse_y do
+    in_compare do
+      sh "bison -r all #{parse_y}"
+      sh "./normalize.rb parse#{v}.output > #{mri_txt}"
+      rm ["parse#{v}.output", "parse#{v}.tab.c"]
+    end
+  end
+
+  file rp_out => :parser
+
+  file c_rp_txt => rp_out do
+    in_compare do
+      sh "./normalize.rb ../#{rp_out} > #{rp_txt}"
+    end
+  end
+
+  compare = "compare#{v}"
+
+  desc "Compare all grammars to MRI"
+  task :compare => compare
+
+  task c_diff => [c_mri_txt, c_rp_txt] do
+    in_compare do
+      system "diff -du #{mri_txt} #{rp_txt} > #{diff}"
+    end
+  end
+
+  desc "Compare #{v} grammar to MRI #{version}"
+  task compare => c_diff do
+    in_compare do
+      system "wc -l #{diff}"
+    end
+  end
+
+  task :clean do
+    rm_f Dir[c_parse_y, c_mri_txt, c_rp_txt]
+  end
+
+  task :realclean do
+    rm_f Dir[tarball]
   end
 end
 
+ruby_parse "1.8.7-p374"
+ruby_parse "1.9.3-p551"
+ruby_parse "2.0.0-p648"
+ruby_parse "2.1.9"
+ruby_parse "2.2.6"
+ruby_parse "2.3.3"
+# TODO ruby_parse "2.4.0"
+
 task :debug => :isolate do
-  ENV["V"] ||= "23"
+  ENV["V"] ||= V1_2.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
@@ -127,22 +199,9 @@ task :debug => :isolate do
   require "ruby_parser"
   require "pp"
 
-  parser = case ENV["V"]
-           when "18" then
-             Ruby18Parser.new
-           when "19" then
-             Ruby19Parser.new
-           when "20" then
-             Ruby20Parser.new
-           when "21" then
-             Ruby21Parser.new
-           when "22" then
-             Ruby22Parser.new
-           when "23" then
-             Ruby23Parser.new
-           else
-             raise "Unsupported version #{ENV["V"]}"
-           end
+  klass = Object.const_get("Ruby#{ENV["V"]}Parser") rescue nil
+  raise "Unsupported version #{ENV["V"]}" unless klass
+  parser = klass.new
 
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
@@ -173,7 +232,7 @@ task :debug_ruby do
 end
 
 task :extract => :isolate do
-  ENV["V"] ||= "19"
+  ENV["V"] ||= V1_2.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
   file = ENV["F"] || ENV["FILE"]

data/compare/normalize.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/ruby -w
+
+good = false
+
+rules = Hash.new { |h,k| h[k] = [] }
+rule = nil
+order = []
+
+def munge s
+  renames = [
+             "'='",             "tEQL",
+             "'!'",             "tBANG",
+             "'%'",             "tPERCENT",
+             "'&'",             "tAMPER2",
+             "'('",             "tLPAREN2",
+             "')'",             "tRPAREN",
+             "'*'",             "tSTAR2",
+             "'+'",             "tPLUS",
+             "','",             "tCOMMA",
+             "'-'",             "tMINUS",
+             "'.'",             "tDOT",
+             "'/'",             "tDIVIDE",
+             "';'",             "tSEMI",
+             "':'",             "tCOLON",
+             "'<'",             "tLT",
+             "'>'",             "tGT",
+             "'?'",             "tEH",
+             "'['",             "tLBRACK",
+             "'\\n'",           "tNL",
+             "']'",             "tRBRACK",
+             "'^'",             "tCARET",
+             "'`'",             "tBACK_REF2",
+             "'{'",             "tLCURLY",
+             "'|'",             "tPIPE",
+             "'}'",             "tRCURLY",
+             "'~'",             "tTILDE",
+             '"["',             "tLBRACK",
+
+             # 2.0 changes?
+             '"<=>"',            "tCMP",
+             '"=="',             "tEQ",
+             '"==="',            "tEQQ",
+             '"!~"',             "tNMATCH",
+             '"=~"',             "tMATCH",
+             '">="',             "tGEQ",
+             '"<="',             "tLEQ",
+             '"!="',             "tNEQ",
+             '"<<"',             "tLSHFT",
+             '">>"',             "tRSHFT",
+             '"*"',              "tSTAR",
+
+             '".."',             "tDOT2",
+
+             '"&"',              "tAMPER",
+             '"&&"',             "tANDOP",
+             '"||"',             "tOROP",
+
+             '"..."',            "tDOT3",
+             '"**"',             "tPOW",
+             '"unary+"',         "tUPLUS",
+             '"unary-"',         "tUMINUS",
+             '"[]"',             "tAREF",
+             '"[]="',            "tASET",
+             '"::"',             "tCOLON2",
+             '"{ arg"',          "tLBRACE_ARG",
+             '"( arg"',          "tLPAREN_ARG",
+             '"("',              "tLPAREN",
+             'rparen',           "tRPAREN",
+             '"{"',              "tLBRACE",
+             '"=>"',             "tASSOC",
+             '"->"',             "tLAMBDA",
+             '":: at EXPR_BEG"', "tCOLON3",
+             '"**arg"',          "tDSTAR",
+             '","',              "tCOMMA",
+
+             # other
+
+             'tLBRACK2',        "tLBRACK", # HACK
+
+             "' '",             "tSPACE", # needs to be later to avoid bad hits
+
+             "/* empty */",     "none",
+             /^\s*$/,           "none",
+             "keyword_BEGIN",   "klBEGIN",
+             "keyword_END",     "klEND",
+             /keyword_(\w+)/,   proc { "k#{$1.upcase}" },
+             /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
+             "kVARIABLE",       "keyword_variable", # ugh
+
+             /@(\d+)\s+/,       "",
+            ]
+
+  renames.each_slice(2) do |(a, b)|
+    if Proc === b then
+      s.gsub!(a, &b)
+    else
+      s.gsub!(a, b)
+    end
+  end
+
+  s.strip
+end
+
+ARGF.each_line do |line|
+  next unless good or line =~ /^-* ?Grammar|\$accept : /
+
+  case line.strip
+  when /^$/ then
+  when /^(\d+) (\$?\w+): (.*)/ then    # yacc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /^(\d+) \s+\| (.*)/ then        # yacc
+    rules[rule] << munge($2)
+  when /^(\d+) (@\d+): (.*)/ then      # yacc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /^rule (\d+) (@?\w+):(.*)/ then # racc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /\$accept/ then                 # byacc?
+    good = true
+  when /Grammar/ then                  # both
+    good = true
+  when /^-+ Symbols/ then              # racc
+    break
+  when /^Terminals/ then               # yacc
+    break
+  when /^\cL/ then                     # byacc
+    break
+  else
+    warn "unparsed: #{$.}: #{line.chomp}"
+  end
+end
+
+require 'yaml'
+
+order.each do |k|
+  next if k =~ /@/
+  puts
+  puts "#{k}:"
+  puts rules[k].map { |r| "    #{r}" }.join "\n"
+end