ruby_parser 3.0.0 → 3.19.1

data/bin/ruby_parse_extract_error

@@ -1,27 +1,27 @@
 #!/usr/bin/ruby -ws
 
-$d ||= false
-$d ||= ENV["DELETE"]
-$t ||= false
-$t ||= ENV["DELETE_TIMEOUT"]
-$q ||= false
-$q ||= ENV["QUIET"]
+$d ||= ENV["DELETE"]         || false
+$t ||= ENV["DELETE_TIMEOUT"] || false
+$m ||= ENV["MOVE_TIMEOUT"]   || false
+$q ||= ENV["QUIET"]          || false
+$s ||= ENV["SPEED"]          || false
 
 require 'rubygems'
 require 'ruby_parser'
+require 'fileutils'
 
 ARGV.push "-" if ARGV.empty?
 
-class Racc::Parser
+class RubyParser
   def extract_defs
-    ss = lexer.src
+    ss = current.lexer.ss
 
     raise "can't access source. possible encoding issue" unless ss
 
     src = ss.string
     pre_error = src[0...ss.pos]
 
-    defs = pre_error.grep(/^ *def/)
+    defs = pre_error.lines.grep(/^ *(?:def|it)/)
 
     raise "can't figure out where the bad code starts" unless defs.last
 
@@ -34,7 +34,7 @@ class Racc::Parser
 
     src = pre_error + post_error[0..idx+$&.length]
 
-    src.scan(/^(( *)def .*?^\2end)/m)
+    src.scan(/^(( *)(?:def|it) .*?^\2end)/m)
   end
 
   def retest_for_errors defs
@@ -56,7 +56,7 @@ def expand path
       files << f if File.file? f
     end
 
-    files
+    files.sort
   else
     Dir.glob path
   end
@@ -82,24 +82,44 @@ def process_error parser
   end
 rescue RuntimeError, Racc::ParseError => e
   warn "# process error: #{e.message.strip}"
+  warn "#   #{e.backtrace.first}"
 end
 
 def process file
-  ruby = file == "-" ? $stdin.read : File.read(file)
+  ruby = file == "-" ? $stdin.binread : File.binread(file)
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   $stderr.print "# Validating #{file}: "
-  parser = Ruby19Parser.new
-  parser.process(ruby, file)
-  warn "good"
+  parser = RubyParser.new
+  t0 = Time.now if $s
+  parser.process(ruby, file, time)
+  if $s then
+    warn "good: #{Time.now - t0}"
+  else
+    warn "good"
+  end
   File.unlink file if $d
 rescue Timeout::Error
   $exit = 1
   warn "TIMEOUT parsing #{file}. Skipping."
-  File.unlink file if $t
+
+  if $m then
+    base_dir, *rest = file.split("/")
+    base_dir.sub!(/\.slow\.?.*/, "")
+    base_dir += ".slow.#{time}"
+
+    new_file = File.join(base_dir, *rest)
+
+    FileUtils.mkdir_p File.dirname(new_file)
+    FileUtils.move file, new_file, verbose:true
+  elsif $t then
+    File.unlink file
+  end
 rescue StandardError, SyntaxError, Racc::ParseError => e
   $exit = 1
   warn ""
   warn "# error: #{e.message.strip}" unless $q
+  warn "#   #{e.backtrace.first}"
   warn ""
   return if $q
 

data/compare/normalize.rb

@@ -0,0 +1,218 @@
+#!/usr/bin/env ruby -w
+
+good = false
+
+rules = Hash.new { |h,k| h[k] = [] }
+rule = nil
+order = []
+
+def munge s
+  renames = [
+             # unquote... wtf?
+             /`(.+?)'/,          proc { $1 },
+             /"'(.+?)'"/,        proc { "\"#{$1}\"" },
+
+             "'='",             "tEQL",
+             "'!'",             "tBANG",
+             "'%'",             "tPERCENT",
+             "'&'",             "tAMPER2",
+             "'('",             "tLPAREN2",
+             "')'",             "tRPAREN",
+             "'*'",             "tSTAR2",
+             "'+'",             "tPLUS",
+             "','",             "tCOMMA",
+             "'-'",             "tMINUS",
+             "'.'",             "tDOT",
+             "'/'",             "tDIVIDE",
+             "';'",             "tSEMI",
+             "':'",             "tCOLON",
+             "'<'",             "tLT",
+             "'>'",             "tGT",
+             "'?'",             "tEH",
+             "'['",             "tLBRACK",
+             "'\\n'",           "tNL",
+             "']'",             "tRBRACK",
+             "'^'",             "tCARET",
+             "'`'",             "tBACK_REF2",
+             "'{'",             "tLCURLY",
+             "'|'",             "tPIPE",
+             "'}'",             "tRCURLY",
+             "'~'",             "tTILDE",
+             '"["',             "tLBRACK",
+
+             # 2.0 changes?
+             '"<=>"',            "tCMP",
+             '"=="',             "tEQ",
+             '"==="',            "tEQQ",
+             '"!~"',             "tNMATCH",
+             '"=~"',             "tMATCH",
+             '">="',             "tGEQ",
+             '"<="',             "tLEQ",
+             '"!="',             "tNEQ",
+             '"<<"',             "tLSHFT",
+             '">>"',             "tRSHFT",
+             '"*"',              "tSTAR",
+
+             '".."',             "tDOT2",
+
+             '"&"',              "tAMPER",
+             '"&&"',             "tANDOP",
+             '"&."',             "tLONELY",
+             '"||"',             "tOROP",
+
+             '"..."',            "tDOT3",
+             '"**"',             "tPOW",
+             '"unary+"',         "tUPLUS",
+             '"unary-"',         "tUMINUS",
+             '"[]"',             "tAREF",
+             '"[]="',            "tASET",
+             '"::"',             "tCOLON2",
+             '"{ arg"',          "tLBRACE_ARG",
+             '"( arg"',          "tLPAREN_ARG",
+             '"("',              "tLPAREN",
+             'rparen',           "tRPAREN",
+             '"{"',              "tLBRACE",
+             '"=>"',             "tASSOC",
+             '"->"',             "tLAMBDA",
+             '":: at EXPR_BEG"', "tCOLON3",
+             '"**arg"',          "tDSTAR",
+             '","',              "tCOMMA",
+
+             # other
+
+             'tLBRACK2',        "tLBRACK", # HACK
+
+             "' '",             "tSPACE", # needs to be later to avoid bad hits
+
+             "%empty",          "none", # newer bison
+             "/* empty */",     "none",
+             /^\s*$/,           "none",
+
+             "keyword_BEGIN",   "klBEGIN",
+             "keyword_END",     "klEND",
+             /keyword_(\w+)/,   proc { "k#{$1.upcase}" },
+             /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
+             "kVARIABLE",       "keyword_variable", # ugh
+
+             # 2.6 collapses klBEGIN to kBEGIN
+             "klBEGIN",   "kBEGIN",
+             "klEND",     "kEND",
+
+             /keyword_(\w+)/,          proc { "k#{$1.upcase}" },
+             /\bk_([^_][a-z_]+)/,      proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,         proc { "k#{$1.upcase}_MOD" },
+
+             "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
+
+             # 2.7 changes:
+
+             '"global variable"',          "tGVAR",
+             '"operator-assignment"',      "tOP_ASGN",
+             '"back reference"',           "tBACK_REF",
+             '"numbered reference"',       "tNTH_REF",
+             '"local variable or method"', "tIDENTIFIER",
+             '"constant"',                 "tCONSTANT",
+
+             '"(.."',                  "tBDOT2",
+             '"(..."',                 "tBDOT3",
+             '"char literal"',         "tCHAR",
+             '"literal content"',      "tSTRING_CONTENT",
+             '"string literal"',       "tSTRING_BEG",
+             '"symbol literal"',       "tSYMBEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+
+             '"float literal"',        "tFLOAT",
+             '"imaginary literal"',    "tIMAGINARY",
+             '"integer literal"',      "tINTEGER",
+             '"rational literal"',     "tRATIONAL",
+
+             '"instance variable"',  "tIVAR",
+             '"class variable"',     "tCVAR",
+             '"terminator"',         "tSTRING_END", # TODO: switch this?
+             '"method"',             "tFID",
+             '"}"',                  "tSTRING_DEND",
+
+             '"do for block"',     "kDO_BLOCK",
+             '"do for condition"', "kDO_COND",
+             '"do for lambda"',    "kDO_LAMBDA",
+             "tLABEL",             "kLABEL",
+
+             # UGH
+             "k_LINE__",       "k__LINE__",
+             "k_FILE__",       "k__FILE__",
+             "k_ENCODING__",   "k__ENCODING__",
+
+             '"defined?"',     "kDEFINED",
+
+             '"do (for condition)"', "kDO_COND",
+             '"do (for lambda)"',    "kDO_LAMBDA",
+             '"do (for block)"',     "kDO_BLOCK",
+
+             /\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,                proc { |x| "k#{$1.upcase}" },
+
+             /\$?@(\d+)(\s+|$)/,    "", # newer bison
+
+             # TODO: remove for 3.0 work:
+             "lex_ctxt ", "" # 3.0 production that's mostly noise right now
+            ]
+
+  renames.each_slice(2) do |(a, b)|
+    if Proc === b then
+      s.gsub!(a, &b)
+    else
+      s.gsub!(a, b)
+    end
+  end
+
+  s.strip
+end
+
+ARGF.each_line do |line|
+  next unless good or line =~ /^-* ?Grammar|\$accept : /
+
+  case line.strip
+  when /^$/ then
+  when /^(\d+) (\$?[@\w]+): (.*)/ then    # yacc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /^(\d+) \s+\| (.*)/ then        # yacc
+    rules[rule] << munge($2)
+  when /^(\d+) (@\d+): (.*)/ then      # yacc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /^rule (\d+) (@?\w+):(.*)/ then # racc
+    rule = $2
+    order << rule unless rules.has_key? rule
+    rules[rule] << munge($3)
+  when /\$accept/ then                 # byacc?
+    good = true
+  when /Grammar/ then                  # both
+    good = true
+  when /^-+ Symbols/ then              # racc
+    break
+  when /^Terminals/ then               # yacc
+    break
+  when /^\cL/ then                     # byacc
+    break
+  else
+    warn "unparsed: #{$.}: #{line.strip.inspect}"
+  end
+end
+
+require 'yaml'
+
+order.each do |k|
+  next if k =~ /@/
+  puts
+  puts "#{k}:"
+  puts rules[k].map { |r| "    #{r}" }.join "\n"
+end

data/debugging.md

@@ -0,0 +1,190 @@
+# Quick Notes to Help with Debugging
+
+## Reducing
+
+One of the most important steps is reducing the code sample to a
+minimal reproduction. For example, one thing I'm debugging right now
+was reported as:
+
+```ruby
+a, b, c, d, e, f, g, h, i, j = 1, *[p1, p2, p3], *[p1, p2, p3], *[p4, p5, p6]
+```
+
+This original sample has 10 items on the left-hand-side (LHS) and 1 +
+3 groups of 3 (calls) on the RHS + 3 arrays + 3 splats. That's a lot.
+
+It's already been reported (perhaps incorrectly) that this has to do
+with multiple splats on the RHS, so let's focus on that. At a minimum
+the code can be reduced to 2 splats on the RHS and some
+experimentation shows that it needs a non-splat item to fail:
+
+```
+_, _, _ = 1, *[2], *[3]
+```
+
+and some intuition further removed the arrays:
+
+```
+_, _, _ = 1, *2, *3
+```
+
+the difference is huge and will make a ton of difference when
+debugging.
+
+## Getting something to compare
+
+```
+% rake debug3 F=file.rb
+```
+
+TODO
+
+## Comparing against ruby / ripper:
+
+```
+% rake cmp3 F=file.rb
+```
+
+This compiles the parser & lexer and then parses file.rb using both
+ruby, ripper, and ruby_parser in debug modes. The output is munged to
+be as uniform as possible and diffable. I'm using emacs'
+`ediff-files3` to compare these files (via `rake cmp3`) all at once,
+but regular `diff -u tmp/{ruby,rp}` will suffice for most tasks.
+
+From there? Good luck. I'm currently trying to backtrack from rule
+reductions to state change differences. I'd like to figure out a way
+to go from this sort of diff to a reasonable test that checks state
+changes but I don't have that set up at this point.
+
+## Adding New Grammar Productions
+
+Ruby adds stuff to the parser ALL THE TIME. It's actually hard to keep
+up with, but I've added some tools and shown what a typical workflow
+looks like. Let's say you want to add ruby 2.7's "beginless range" (eg
+`..42`).
+
+Whenever there's a language feature missing, I start with comparing
+the parse trees between MRI and RP:
+
+### Structural Comparing
+
+There's a bunch of rake tasks `compare27`, `compare26`, etc that try
+to normalize and diff MRI's parse.y parse tree (just the structure of
+the tree in yacc) to ruby\_parser's parse tree (racc). It's the first
+thing I do when I'm adding a new version. Stub out all the version
+differences, and then start to diff the structure and move
+ruby\_parser towards the new changes.
+
+Some differences are just gonna be there... but here's an example of a
+real diff between MRI 2.7 and ruby_parser as of today:
+
+```diff
+     arg tDOT3 arg
+     arg tDOT2
+     arg tDOT3
+-    tBDOT2 arg
+-    tBDOT3 arg
+     arg tPLUS arg
+     arg tMINUS arg
+     arg tSTAR2 arg
+```
+
+This is a new language feature that ruby_parser doesn't handle yet.
+It's in MRI (the left hand side of the diff) but not ruby\_parser (the
+right hand side) so it is a `-` or missing line.
+
+Some other diffs will have both `+` and `-` lines. That usually
+happens when MRI has been refactoring the grammar. Sometimes I choose
+to adapt those refactorings and sometimes it starts to get too
+difficult to maintain multiple versions of ruby parsing in a single
+file.
+
+But! This structural comparing is always a place you should look when
+ruby_parser is failing to parse something. Maybe it just hasn't been
+implemented yet and the easiest place to look is the diff.
+
+### Starting Test First
+
+The next thing I do is to add a parser test to cover that feature. I
+usually start with the parser and work backwards towards the lexer as
+needed, as I find it structures things properly and keeps things goal
+oriented.
+
+So, make a new parser test, usually in the versioned section of the
+parser tests.
+
+```
+  def test_beginless2
+    rb = "..10\n; ..a\n; c"
+    pt = s(:block,
+           s(:dot2, nil, s(:lit, 0).line(1)).line(1),
+           s(:dot2, nil, s(:call, nil, :a).line(2)).line(2),
+           s(:call, nil, :c).line(3)).line(1)
+
+    assert_parse_line rb, pt, 1
+
+    flunk "not done yet"
+  end
+```
+
+(In this case copied and modified the tests for open ranges from 2.6)
+and run it to get my first error:
+
+```
+% rake N=/beginless/
+
+...
+
+E
+
+Finished in 0.021814s, 45.8421 runs/s, 0.0000 assertions/s.
+
+  1) Error:
+TestRubyParserV27#test_whatevs:
+Racc::ParseError: (string):1 :: parse error on value ".." (tDOT2)
+    GEMS/2.7.0/gems/racc-1.5.0/lib/racc/parser.rb:538:in `on_error'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1304:in `on_error'
+    (eval):3:in `_racc_do_parse_c'
+    (eval):3:in `do_parse'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1329:in `block in process'
+    RUBY/lib/ruby/2.7.0/timeout.rb:95:in `block in timeout'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `block in catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:33:in `catch'
+    RUBY/lib/ruby/2.7.0/timeout.rb:110:in `timeout'
+    WORK/ruby_parser/dev/lib/ruby_parser_extras.rb:1317:in `process'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4198:in `assert_parse'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4221:in `assert_parse_line'
+    WORK/ruby_parser/dev/test/test_ruby_parser.rb:4451:in `test_whatevs'
+```
+
+For starters, we know the missing production is for `tBDOT2 arg`. It
+is currently blowing up because it is getting `tDOT2` and simply
+doesn't know what to do with it, so it raises the error. As the diff
+suggests, that's the wrong token to begin with, so it is probably time
+to also create a lexer test:
+
+```
+def test_yylex_bdot2
+  assert_lex3("..42",
+              s(:dot2, nil, s(:lit, 42)),
+
+              :tBDOT2,   "..", EXPR_BEG,
+              :tINTEGER, "42", EXPR_NUM)
+
+  flunk "not done yet"
+end
+```
+
+This one is mostly speculative at this point. It says "if we're lexing
+this string, we should get this sexp if we fully parse it, and the
+lexical stream should look like this"... That last bit is mostly made
+up at this point. Sometimes I don't know exactly what expression state
+things should be in until I start really digging in.
+
+At this point, I have 2 failing tests that are directing me in the
+right direction. It's now a matter of digging through
+`compare/parse26.y` to see how the lexer differs and implementing
+it...
+
+But this is a good start to the doco for now. I'll add more later.

data/gauntlet.md

@@ -0,0 +1,107 @@
+# Running the Gauntlet
+
+## Maintaining a Gem Mirror
+
+I use rubygems-mirror to keep an archive of all the latest rubygems on
+an external disk. Here is the config:
+
+```
+---
+- from: https://rubygems.org
+  to: /Volumes/StuffA/gauntlet/mirror
+  parallelism: 10
+  retries: 3
+  delete: true
+  skiperror: true
+  hashdir: true
+```
+
+And I update using rake:
+
+```
+% cd GIT/rubygems/rubygems-mirror
+% git down
+% rake mirror:latest
+% /Volumes/StuffA/gauntlet/bin/cleanup.rb -y -v
+```
+
+This rather quickly updates my mirror to the latest versions of
+everything and then deletes all old versions. I then run a cleanup
+script that fixes the file dates to their publication date and deletes
+any gems that have invalid specs. This can argue with the mirror a
+bit, but it is pretty minimal (currently ~20 bad gems).
+
+## Curating an Archive of Ruby Files
+
+Next, I process the gem mirror into a much more digestable structure
+using `unpack_gems.rb`.
+
+```
+% cd RP/gauntlet
+% time caffeinate /Volumes/StuffA/gauntlet/bin/unpack_gems.rb -v [-a] ; say done
+... waaaait ...
+% DIR=gauntlet.$(today).(all|new).noindex
+% mv hashed.noindex $DIR
+% tar vc -T <(fd -tf . $DIR | sort) | zstd -5 -T0 --long > archives/$DIR.tar.zst ; say done
+% ./bin/sync.sh
+```
+
+This script filters all the newer (< 1 year old) gems (unless `-a` is
+used), unpacks them, finds all the files that look like they're valid
+ruby, ensures they're valid ruby (using the current version of ruby to
+compile them), and then moves them into a SHA dir structure that looks
+something like this:
+
+```
+hashed.noindex/a/b/c/<full_file_sha>.rb
+```
+
+This removes all duplicates and puts everything in a fairly even,
+wide, flat directory layout.
+
+This process takes a very long time, even with a lot of
+parallelization. There are currently about 160k gems in the mirror.
+Unpacking, validating, SHA'ing everything is disk and CPU intensive.
+The `.noindex` extension stops spotlight from indexing the continous
+churn of files being unpacked and moved and saves time.
+
+Finally, I rename and archive it all up (currently using zstd to
+compress).
+
+### Stats
+
+```
+9696 % find gauntlet.$(today).noindex -type f | lc
+  561270
+3.5G gauntlet.2021-08-06.noindex
+239M gauntlet.2021-08-06.noindex.tar.zst
+```
+
+So I wind up with a little over half a million unique ruby files to
+parse. It's about 3.5g but compresses very nicely down to 240m
+
+## Running the Gauntlet
+
+Assuming you're starting from scratch, unpack the archive once:
+
+```
+% zstdcat gauntlet.$(today).noindex.tar.zst | tar x
+```
+
+Then, either run a single process (easier to read):
+
+```
+% ./gauntlet/bin/gauntlet.rb gauntlet/*.noindex/?
+```
+
+Or max out your machine using xargs (note the `-P 16` and choose accordingly):
+
+```
+% ls -d gauntlet/*.noindex/?/? | time xargs -n 1 -P 16 ./gauntlet/bin/gauntlet.rb
+```
+
+In another terminal I usually monitor the progress like so:
+
+```
+% while true ; do clear; fd . -t d -t e gauntlet/*.noindex -X rmdir -p 2> /dev/null ; for D in gauntlet/*.noindex/? ; do echo -n "$D: "; fd .rb $D | wc -l ; done ; echo ; sleep 30 ; done
+```

data/lib/.document

@@ -0,0 +1 @@
+*.rb

data/lib/rp_extensions.rb

@@ -0,0 +1,53 @@
+# :stopdoc:
+# WHY do I have to do this?!?
+class Regexp
+  ONCE = 0 unless defined? ONCE # FIX: remove this - it makes no sense
+
+  unless defined? ENC_NONE then
+    ENC_NONE = /x/n.options
+    ENC_EUC  = /x/e.options
+    ENC_SJIS = /x/s.options
+    ENC_UTF8 = /x/u.options
+  end
+end
+# :startdoc:
+
+class Array
+  def prepend *vals
+    self[0,0] = vals
+  end
+end unless [].respond_to?(:prepend)
+
+# :stopdoc:
+class Symbol
+  def end_with? o
+    self.to_s.end_with? o
+  end
+end unless :woot.respond_to?(:end_with?)
+# :startdoc:
+
+############################################################
+# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
+
+class String
+  def clean_caller
+    self.sub(File.dirname(__FILE__), "./lib").sub(/:in.*/, "")
+  end if $DEBUG
+end
+
+require "sexp"
+
+class Sexp
+  attr_writer :paren # TODO: retire
+
+  def paren
+    @paren ||= false
+  end
+
+  def block_pass?
+    any? { |s| Sexp === s && s.sexp_type == :block_pass }
+  end
+end
+
+# END HACK
+############################################################