ruby_parser 3.14.2 → 3.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 397471879b6b8dc9138464e3c08e585fbbf6c314c664573c660d9f9853e8b503
-  data.tar.gz: e90bc3750fac22cefb540a1d847cd8f07af611096b8095feaa17885fe040dff7
+  metadata.gz: ff6be95e278654e341f5279fed2fd7f0c9a96d93b2fd23ba1ff4b181d593be18
+  data.tar.gz: ab91b782eb2e77cdd855fa68f4699614b6160ebcca623dd8be25719b410b4206
 SHA512:
-  metadata.gz: 54edcdda388c8ad13123ae58484e2ab9c0f3eb49893de14f947751cd65133d6dbb5805eb9baa22f6c91f18e20c759053d98265415128b13a107f6a652b1459a7
-  data.tar.gz: c59425b19b4017e420963aaf37a27bfa7ae31e15d2824efd3bbc0dcf3590f6f3a76cbeee22c62204e1d45fc1fa4759d478dde8cd5f1984007071217659450f42
+  metadata.gz: a469da9dadd1eeb35a48dbb34548e70feed8ca83b2d27e41c6bf940cf9dd779622fbddcc4b3c50534f46c6de42f1f085754739d76051413866ee6557fe84050d
+  data.tar.gz: d182a507b167a6c9af4a7a48e748f55066462888b646a38a08ab12c4365a57645c5216baa2d64c99c8d7b6ebcb4e4b22219a9da9a7d0bd945c00fc21104d7343

data/History.rdoc

@@ -1,3 +1,63 @@
+=== 3.16.0 / 2021-05-15
+
+* 1 major enhancement:
+
+  * Added tentative 3.0 support.
+
+* 3 minor enhancements:
+
+  * Added lexing for "beginless range" (bdots).
+  * Added parsing for bdots.
+  * Updated rake compare task to download xz files, bumped versions, etc
+
+* 4 bug fixes:
+
+  * Bump rake dependency to >= 10, < 15. (presidentbeef)
+  * Bump sexp_processor dependency to 4.15.1+. (pravi)
+  * Fixed minor state mismatch at the end of parsing to make diffing a little cleaner.
+  * Fixed normalizer to deal with new bison token syntax
+
+=== 3.15.1 / 2021-01-10
+
+* 1 bug fix:
+
+  * Bumped ruby version to include < 4 (trunk).
+
+=== 3.15.0 / 2020-08-31
+
+* 1 major enhancement:
+
+  * Added tentative 2.7 support.
+
+* 1 minor enhancement:
+
+  * Improved ruby_parse_extract_error's handling of moving slow files out.
+
+* 22 bug fixes:
+
+  * Bumped ruby version to include 3.0 (trunk).
+  * Fix an error related to empty ensure bodies. (presidentbeef)
+  * Fix handling of bad magic encoding comment.
+  * Fixed SystemStackError when parsing a huoooge hash, caused by a splat arg.
+  * Fixed a number of errors parsing do blocks in strange edge cases.
+  * Fixed a string backslash lexing bug when the string is an invalid encoding. (nijikon, gmcgibbon)
+  * Fixed bug assigning line number to some arg nodes.
+  * Fixed bug concatinating string literals with differing encodings.
+  * Fixed bug lexing heredoc w/ nasty mix of \r\n and \n.
+  * Fixed bug lexing multiple codepoints in \u{0000 1111 2222} forms.
+  * Fixed bug setting line numbers in empty xstrings in some contexts.
+  * Fixed edge case on call w/ begin + do block as an arg.
+  * Fixed handling of UTF BOM.
+  * Fixed handling of lexer state across string interpolation braces.
+  * Fixed infinite loop when lexing backslash+cr+newline (aka dos-files)
+  * Fixed lambda + do block edge case.
+  * Fixed lexing of some ?\M... and ?\C... edge cases.
+  * Fixed more do/brace block edge case failures.
+  * Fixed parsing bug where splat was used in the middle of a list.
+  * Fixed parsing of interpolation in heredoc-like strings. (presidentbeef)
+  * Fixed parsing some esoteric edge cases in op_asgn.
+  * Fixed unicode processing in ident chars so now they better mix.
+
 === 3.14.2 / 2020-02-06
 
 * 1 minor enhancement:

data/Manifest.txt

@@ -24,6 +24,10 @@ lib/ruby25_parser.rb
 lib/ruby25_parser.y
 lib/ruby26_parser.rb
 lib/ruby26_parser.y
+lib/ruby27_parser.rb
+lib/ruby27_parser.y
+lib/ruby30_parser.rb
+lib/ruby30_parser.y
 lib/ruby_lexer.rb
 lib/ruby_lexer.rex
 lib/ruby_lexer.rex.rb

data/Rakefile

@@ -8,11 +8,12 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 Hoe.plugin :rdoc
 
+Hoe.add_include_dirs "lib"
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
 
-V2   = %w[20 21 22 23 24 25 26]
+V2   = %w[20 21 22 23 24 25 26 27 30]
 V2.replace [V2.last] if ENV["FAST"] # HACK
 
 Hoe.spec "ruby_parser" do
@@ -20,11 +21,18 @@ Hoe.spec "ruby_parser" do
 
   license "MIT"
 
-  dependency "sexp_processor", "~> 4.9"
-  dependency "rake", "< 11", :developer
+  dependency "sexp_processor", ["~> 4.15", ">= 4.15.1"]
+  dependency "rake", [">= 10", "< 15"], :developer
   dependency "oedipus_lex", "~> 2.5", :developer
 
-  require_ruby_version "~> 2.2"
+  # NOTE: Ryan!!! Stop trying to fix this dependency! Isolate just
+  # can't handle having a faux-gem half-installed! Stop! Just `gem
+  # install racc` and move on. Revisit this ONLY once racc-compiler
+  # gets split out.
+
+  dependency "racc", "~> 1.5", :developer
+
+  require_ruby_version [">= 2.1", "< 4"]
 
   if plugin? :perforce then     # generated files
     V2.each do |n|
@@ -56,6 +64,8 @@ end
 
 file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
+task :generate => [:lexer, :parser]
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["diff.diff"] + # not all diffs. bit me too many times
@@ -89,7 +99,7 @@ end
 
 def dl v
   dir = v[/^\d+\.\d+/]
-  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.bz2"
+  url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.xz"
   path = File.basename url
   unless File.exist? path then
     system "curl -O #{url}"
@@ -101,7 +111,7 @@ def ruby_parse version
   rp_txt    = "rp#{v}.txt"
   mri_txt   = "mri#{v}.txt"
   parse_y   = "parse#{v}.y"
-  tarball   = "ruby-#{version}.tar.bz2"
+  tarball   = "ruby-#{version}.tar.xz"
   ruby_dir  = "ruby-#{version}"
   diff      = "diff#{v}.diff"
   rp_out    = "lib/ruby#{v}_parser.output"
@@ -121,23 +131,40 @@ def ruby_parse version
     end
   end
 
+  desc "fetch all tarballs"
+  task :fetch => c_tarball
+
   file c_parse_y => c_tarball do
     in_compare do
-      system "tar yxf #{tarball} #{ruby_dir}/{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+      extract_glob = case version
+                     when /2\.7|3\.0/
+                       "{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
+                     else
+                       "{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
+                     end
+      system "tar Jxf #{tarball} #{ruby_dir}/#{extract_glob}"
+
       Dir.chdir ruby_dir do
         if File.exist? "tool/id2token.rb" then
           sh "ruby tool/id2token.rb --path-separator=.:./ id.h parse.y | expand > ../#{parse_y}"
         else
           sh "expand parse.y > ../#{parse_y}"
         end
+
+        ruby "-pi", "-e", 'gsub(/^%define\s+api\.pure/, "%pure-parser")', "../#{parse_y}"
       end
       sh "rm -rf #{ruby_dir}"
     end
   end
 
+  bison = Dir["/opt/homebrew/opt/bison/bin/bison",
+              "/usr/local/opt/bison/bin/bison",
+              `which bison`.chomp,
+             ].first
+
   file c_mri_txt => [c_parse_y, normalize] do
     in_compare do
-      sh "bison -r all #{parse_y}"
+      sh "#{bison} -r all #{parse_y}"
       sh "./normalize.rb parse#{v}.output > #{mri_txt}"
       rm ["parse#{v}.output", "parse#{v}.tab.c"]
     end
@@ -178,20 +205,54 @@ def ruby_parse version
   end
 end
 
+task :versions do
+  require "open-uri"
+  require "net/http" # avoid require issues in threads
+  require "net/https"
+
+  versions = %w[ 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 ]
+
+  base_url = "https://cache.ruby-lang.org/pub/ruby"
+
+  class Array
+    def human_sort
+      sort_by { |item| item.to_s.split(/(\d+)/).map { |e| [e.to_i, e] } }
+    end
+  end
+
+  versions = versions.map { |ver|
+    Thread.new {
+      URI
+        .parse("#{base_url}/#{ver}/")
+        .read
+        .scan(/ruby-\d+\.\d+\.\d+[-\w.]*?.tar.gz/)
+        .reject { |s| s =~ /-(?:rc|preview)\d/ }
+        .human_sort
+        .last
+        .delete_prefix("ruby-")
+        .delete_suffix ".tar.gz"
+    }
+  }.map(&:value).sort
+
+  puts versions.map { |v| "ruby_parse %p" % [v] }
+end
+
 ruby_parse "2.0.0-p648"
-ruby_parse "2.1.9"
-ruby_parse "2.2.9"
+ruby_parse "2.1.10"
+ruby_parse "2.2.10"
 ruby_parse "2.3.8"
-ruby_parse "2.4.5"
-ruby_parse "2.5.3"
-ruby_parse "2.6.1"
+ruby_parse "2.4.10"
+ruby_parse "2.5.9"
+ruby_parse "2.6.8"
+ruby_parse "2.7.4"
+ruby_parse "3.0.2"
 
 task :debug => :isolate do
   ENV["V"] ||= V2.last
   Rake.application[:parser].invoke # this way we can have DEBUG set
   Rake.application[:lexer].invoke # this way we can have DEBUG set
 
-  $: << "lib"
+  $:.unshift "lib"
   require "ruby_parser"
   require "pp"
 
@@ -214,8 +275,9 @@ task :debug => :isolate do
 
   begin
     pp parser.process(ruby, file, time)
-  rescue Racc::ParseError => e
+  rescue ArgumentError, Racc::ParseError => e
     p e
+    puts e.backtrace.join "\n  "
     ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
@@ -232,12 +294,17 @@ task :debug3 do
 
   ENV.delete "V"
 
+  sh "ruby -v"
   sh "ruby -y #{file} 2>&1 | #{munge} > tmp/ruby"
   sh "./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
-  sh "rake debug F=#{file} DEBUG=1 V=25 2>&1 | #{munge} > tmp/rp"
+  sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
   sh "diff -U 999 -d tmp/{rip,rp}"
 end
 
+task :cmp do
+  sh %(emacsclient --eval '(ediff-files "tmp/ruby" "tmp/rp")')
+end
+
 task :cmp3 do
   sh %(emacsclient --eval '(ediff-files3 "tmp/ruby" "tmp/rip" "tmp/rp")')
 end

data/bin/ruby_parse_extract_error

@@ -104,9 +104,14 @@ rescue Timeout::Error
   warn "TIMEOUT parsing #{file}. Skipping."
 
   if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
   elsif $t then
     File.unlink file
   end

data/compare/normalize.rb

@@ -8,6 +8,10 @@ order = []
 
 def munge s
   renames = [
+             # unquote... wtf?
+             /`(.+?)'/,          proc { $1 },
+             /"'(.+?)'"/,        proc { "\"#{$1}\"" },
+
              "'='",             "tEQL",
              "'!'",             "tBANG",
              "'%'",             "tPERCENT",
@@ -100,6 +104,43 @@ def munge s
 
              "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
 
+             # 2.7 changes:
+
+             '"global variable"',          "tGVAR",
+             '"operator-assignment"',      "tOP_ASGN",
+             '"back reference"',           "tBACK_REF",
+             '"numbered reference"',       "tNTH_REF",
+             '"local variable or method"', "tIDENTIFIER",
+             '"constant"',                 "tCONSTANT",
+
+             '"(.."',                  "tBDOT2",
+             '"(..."',                 "tBDOT3",
+             '"char literal"',         "tCHAR",
+             '"literal content"',      "tSTRING_CONTENT",
+             '"string literal"',       "tSTRING_BEG",
+             '"symbol literal"',       "tSYMBEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+
+             '"float literal"',        "tFLOAT",
+             '"imaginary literal"',    "tIMAGINARY",
+             '"integer literal"',      "tINTEGER",
+             '"rational literal"',     "tRATIONAL",
+
+             '"instance variable"',  "tIVAR",
+             '"class variable"',     "tCVAR",
+             '"terminator"',         "tSTRING_END", # TODO: switch this?
+             '"method"',             "tFID",
+             '"}"',                  "tSTRING_DEND",
+
+             '"do for block"',     "kDO_BLOCK",
+             '"do for condition"', "kDO_COND",
+             '"do for lambda"',    "kDO_LAMBDA",
+
              # UGH
              "k_LINE__",       "k__LINE__",
              "k_FILE__",       "k__FILE__",
@@ -107,13 +148,12 @@ def munge s
 
              '"defined?"',     "kDEFINED",
 
-
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
 
-             /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
-             /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
             ]
@@ -134,7 +174,7 @@ ARGF.each_line do |line|
 
   case line.strip
   when /^$/ then
-  when /^(\d+) (\$?\w+): (.*)/ then    # yacc
+  when /^(\d+) (\$?[@\w]+): (.*)/ then    # yacc
     rule = $2
     order << rule unless rules.has_key? rule
     rules[rule] << munge($3)
@@ -159,7 +199,7 @@ ARGF.each_line do |line|
   when /^\cL/ then                     # byacc
     break
   else
-    warn "unparsed: #{$.}: #{line.chomp}"
+    warn "unparsed: #{$.}: #{line.strip.inspect}"
   end
 end
 

data/debugging.md

@@ -1,5 +1,44 @@
 # Quick Notes to Help with Debugging
 
+## Reducing
+
+One of the most important steps is reducing the code sample to a
+minimal reproduction. For example, one thing I'm debugging right now
+was reported as:
+
+```ruby
+a, b, c, d, e, f, g, h, i, j = 1, *[p1, p2, p3], *[p1, p2, p3], *[p4, p5, p6]
+```
+
+This original sample has 10 items on the left-hand-side (LHS) and 1 +
+3 groups of 3 (calls) on the RHS + 3 arrays + 3 splats. That's a lot.
+
+It's already been reported (perhaps incorrectly) that this has to do
+with multiple splats on the RHS, so let's focus on that. At a minimum
+the code can be reduced to 2 splats on the RHS and some
+experimentation shows that it needs a non-splat item to fail:
+
+```
+_, _, _ = 1, *[2], *[3]
+```
+
+and some intuition further removed the arrays:
+
+```
+_, _, _ = 1, *2, *3
+```
+
+the difference is huge and will make a ton of difference when
+debugging.
+
+## Getting something to compare
+
+```
+% rake debug3 F=file.rb
+```
+
+TODO
+
 ## Comparing against ruby / ripper:
 
 ```
@@ -16,3 +55,136 @@ From there? Good luck. I'm currently trying to backtrack from rule
 reductions to state change differences. I'd like to figure out a way
 to go from this sort of diff to a reasonable test that checks state
 changes but I don't have that set up at this point.
+
+## Adding New Grammar Productions
+
+Ruby adds stuff to the parser ALL THE TIME. It's actually hard to keep
+up with, but I've added some tools and shown what a typical workflow
+looks like. Let's say you want to add ruby 2.7's "beginless range" (eg
+`..42`).
+
+Whenever there's a language feature missing, I start with comparing
+the parse trees between MRI and RP:
+
+### Structural Comparing
+
+There's a bunch of rake tasks `compare27`, `compare26`, etc that try
+to normalize and diff MRI's parse.y parse tree (just the structure of
+the tree in yacc) to ruby\_parser's parse tree (racc). It's the first
+thing I do when I'm adding a new version. Stub out all the version
+differences, and then start to diff the structure and move
+ruby\_parser towards the new changes.
+
+Some differences are just gonna be there... but here's an example of a
+real diff between MRI 2.7 and ruby_parser as of today:
+
+```diff
+     arg tDOT3 arg
+     arg tDOT2
+     arg tDOT3
+-    tBDOT2 arg
+-    tBDOT3 arg
+     arg tPLUS arg
+     arg tMINUS arg
+     arg tSTAR2 arg
+```
+
+This is a new language feature that ruby_parser doesn't handle yet.
+It's in MRI (the left hand side of the diff) but not ruby\_parser (the
+right hand side) so it is a `-` or missing line.
+
+Some other diffs will have both `+` and `-` lines. That usually
+happens when MRI has been refactoring the grammar. Sometimes I choose
+to adapt those refactorings and sometimes it starts to get too
+difficult to maintain multiple versions of ruby parsing in a single
+file.
+
+But! This structural comparing is always a place you should look when
+ruby_parser is failing to parse something. Maybe it just hasn't been
+implemented yet and the easiest place to look is the diff.
+
+### Starting Test First
+
+The next thing I do is to add a parser test to cover that feature. I
+usually start with the parser and work backwards towards the lexer as
+needed, as I find it structures things properly and keeps things goal
+oriented.
+
+So, make a new parser test, usually in the versioned section of the
+parser tests.
+
+```
+  def test_beginless2
+    rb = "..10\n; ..a\n; c"
+    pt = s(:block,
+           s(:dot2, nil, s(:lit, 0).line(1)).line(1),
+           s(:dot2, nil, s(:call, nil, :a).line(2)).line(2),
+           s(:call, nil, :c).line(3)).line(1)
+
+    assert_parse_line rb, pt, 1
+
+    flunk "not done yet"
+  end
+```
+
+(In this case copied and modified the tests for open ranges from 2.6)
+and run it to get my first error:
+
+```
+% rake N=/beginless/
+
+...
+
+E
+
+Finished in 0.021814s, 45.8421 runs/s, 0.0000 assertions/s.
+
+  1) Error:
+TestRubyParserV27#test_whatevs:
+Racc::ParseError: (string):1 :: parse error on value ".." (tDOT2)
+    GEMS/2.7.0/gems/racc-1.5.0/lib/racc/parser.rb:538:in `on_error'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1304:in `on_error'
+    (eval):3:in `_racc_do_parse_c'
+    (eval):3:in `do_parse'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1329:in `block in process'
+    RUBY/lib/ruby/2.7.0/timeout.rb:95:in `block in timeout'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `block in catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:110:in `timeout'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1317:in `process'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4198:in `assert_parse'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4221:in `assert_parse_line'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4451:in `test_whatevs'
+```
+
+For starters, we know the missing production is for `tBDOT2 arg`. It
+is currently blowing up because it is getting `tDOT2` and simply
+doesn't know what to do with it, so it raises the error. As the diff
+suggests, that's the wrong token to begin with, so it is probably time
+to also create a lexer test:
+
+```
+def test_yylex_bdot2
+  assert_lex3("..42",
+              s(:dot2, nil, s(:lit, 42)),
+
+              :tBDOT2,   "..", EXPR_BEG,
+              :tINTEGER, "42", EXPR_NUM)
+
+  flunk "not done yet"
+end
+```
+
+This one is mostly speculative at this point. It says "if we're lexing
+this string, we should get this sexp if we fully parse it, and the
+lexical stream should look like this"... That last bit is mostly made
+up at this point. Sometimes I don't know exactly what expression state
+things should be in until I start really digging in.
+
+At this point, I have 2 failing tests that are directing me in the
+right direction. It's now a matter of digging through
+`compare/parse26.y` to see how the lexer differs and implementing
+it...
+
+But this is a good start to the doco for now. I'll add more later.