ruby_parser 3.12.0 → 3.18.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b5e0623385305074b5cbba3e5aaa136b05cc273738be87825977b539773546c
-  data.tar.gz: 473424dfffced52816a9096d4b154f789f6a1cf4aabb3d71e5e2c685fc04928b
+  metadata.gz: dad59e40690dfb746fea4399e13d05f97b43cf562c2559c67d1f9b5868da86c5
+  data.tar.gz: 0c14605ecf34929f6d4865c20f085b6ef2a35533a70e530f261f9d04f44998cb
 SHA512:
-  metadata.gz: dd3c92a567c6b473229d0dd5e152a146f868c36a17e795d2a9a3425700add99d761e371e72e6b92f1dfb38160c79b1d9ef166b7a140f1e805cea2eea70f7c532
-  data.tar.gz: 4f2a1294ea184182574c455aca42adbf34d28273d10b91dbdedb122c54be2c3a685195e18de52ddd70ff8c9fb8c01d6825d9b62d3c3f5a3768fe8d8479c2eeb6
+  metadata.gz: 4fa69e55ac577e55fa2945ef9c135b47359a646e654bc8cf9f2ead973852adb97350cf67ac5da549592ea583f4824cd5f6930a167fa19cf3f914cb6d6888093d
+  data.tar.gz: 0c13eb02ffaf3b3c60ef358be2fa8f70b951e947170d3eeb8a94e5c37bec5dc8282376e4ab3f03de99e49b6e2aac2aa13169b31c30de1067e1a290c1fd65803a

data/.autotest

@@ -7,20 +7,20 @@ require 'autotest/rcov' if ENV['RCOV']
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
   at.libs << ":../../sexp_processor/dev/lib"
-  at.add_exception "unit"
-  at.add_exception "coverage"
-  at.add_exception "coverage.info"
   at.add_exception ".diff"
+  at.add_exception ".rdoc"
+  at.add_exception ".yml"
+  at.add_exception ".txt"
   at.add_exception ".output"
-  at.add_exception "lib/ruby18_parser.rb"
-  at.add_exception "lib/ruby19_parser.rb"
-  at.add_exception "lib/ruby20_parser.rb"
-  at.add_exception "lib/ruby21_parser.rb"
-  at.add_exception "lib/ruby22_parser.rb"
-  at.add_exception "lib/ruby20_parser.y"
-  at.add_exception "lib/ruby21_parser.y"
-  at.add_exception "lib/ruby22_parser.y"
-  at.add_exception "lib/ruby_lexer.rex.rb"
+  at.add_exception "bin"
+  at.add_exception "compare"
+  at.add_exception "coverage"
+  at.add_exception "coverage.info"
+  at.add_exception "misc"
+
+  Dir["lib/ruby??_parser.{rb,y}"].each do |f|
+    at.add_exception f
+  end
 
   Dir["gauntlet*"].each do |f|
     at.add_exception f
@@ -30,12 +30,12 @@ Autotest.add_hook :initialize do |at|
   at.testlib = "minitest/autorun"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
-    g = File.basename(f, ".y").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".y").gsub("_", "_?").gsub(/2\d/, '')
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
   at.add_mapping(/^lib\/.*\.yy$/) do |f, _|
-    g = File.basename(f, ".yy").gsub("_", "_?").gsub(/1[89]|2[012]/, '')
+    g = File.basename(f, ".yy").gsub("_", "_?")
     at.files_matching %r%^test/.*#{g}.rb$%
   end
 
@@ -52,27 +52,16 @@ Autotest.add_hook :initialize do |at|
     at.files_matching(/test_.*rb$/)
   end
 
-  %w(TestEnvironment TestStackState).each do |klass|
+  %w(TestEnvironment TestStackState TestValueExpr).each do |klass|
     at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
   end
 
-  %w(TestRuby18Parser TestRuby19Parser TestRuby20Parser TestRuby21Parser TestRuby22Parser TestRuby23Parser TestParseTree).each do |klass| # HACK
-    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
+  Dir["lib/ruby??_parser.rb"].each do |s|
+    n = s[/\d+/]
+    at.extra_class_map["TestRubyParserV#{n}"] = "test/test_ruby_parser.rb"
   end
 end
 
 Autotest.add_hook :run_command do |at, _|
   system "rake parser lexer DEBUG=1"
 end
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/1.9.3-p194/bin/ruby"
-  end
-end if ENV["R19"]
-
-class Autotest
-  def ruby
-    File.expand_path "~/.multiruby/install/2.0.0-p195/bin/ruby"
-  end
-end if ENV["R20"]

data/History.rdoc

@@ -1,3 +1,286 @@
+=== 3.18.1 / 2021-11-10
+
+* 1 minor enhancement:
+
+  * All parser tests are now explicitly testing line numbers at every level.
+
+* 3 bug fixes:
+
+  * Fixed endless method with noargs. (mitsuru)
+  * Fixed line numbers on some yield forms.
+  * Handle and clearly report if unifdef is missing.
+
+=== 3.18.0 / 2021-10-27
+
+Holy crap... 58 commits! 2.7 and 3.0 are feature complete. Strings
+& heredocs have been rewritten.
+
+* 9 major enhancements:
+
+  * !!! Rewrote lexer (and friends) for strings, heredocs, and %*[] constructs.
+  * Massive overhaul on line numbers.
+  * Freeze input! Finally!!! No more modifying the input string for heredocs.
+  * Overhauled RPStringScanner. Removed OLD compatibility methods!
+  * Removed Sexp methods: value, to_sym, add, add_all, node_type, values.
+    * value moved to sexp_processor.
+  * Removed String#grep monkey-patch.
+  * Removed String#lineno monkey-patch.
+  * Removed string_to_pos, charpos, etc hacks for ancient ruby versions.
+  * Removed unread_many... NO! NO EDITING THE INPUT STRING!
+
+* 31 minor enhancements:
+
+  * 2.7/3.0: many more pattern edge cases
+  * 2.7: Added `mlhs = rhs rescue expr`
+  * 2.7: refactored destructured args (`|(k,v)|`) and unfactored(?!) case_body/args.
+  * 3.0: excessed_comma
+  * 3.0: finished most everything: endless methods, patterns, etc.
+  * 3.0: refactored / added new pattern changes
+  * Added RubyLexer#in_heredoc? (ie, is there old_ss ?)
+  * Added RubyLexer#old_ss and old_lineno and removed much of SSStack(ish).
+  * Added Symbol#end_with? when necessary
+  * Added TALLY and DEBUG options for ss.getch and ss.scan
+  * Added ignore_body_comments to make parser productions more clear.
+  * Added support for no_kwarg (eg `def f(**nil)`).
+  * Added support for no_kwarg in blocks (eg `f { |**nil| }`).
+  * Augmented generated parser files to have frozen_string_literal comments and fixed tests.
+  * Broke out 3.0 parser into its own to ease development.
+  * Bumped dependencies on sexp_processor and oedipus_lex.
+  * Clean generated 3.x files.
+  * Extracted all string scanner methods to their own module.
+  * Fixed some precedence decls.
+  * Implemented most of pattern matching for 2.7+.
+  * Improve lex_state= to report location in verbose debug mode.
+  * Made it easier to debug with a particular version of ruby via rake.
+  * Make sure ripper uses the same version of ruby we specified.
+  * Moved all string/heredoc/etc code to ruby_lexer_strings.rb
+  * Remove warning from newer bisons.
+  * Sprinkled in some frozen_string_literal, but mostly helped by oedipus bump.
+  * Switch to comparing against ruby binary since ripper is buggy.
+  * bugs task should try both bug*.rb and bad*.rb.
+  * endless methods
+  * f_any_kwrest refactoring.
+  * refactored defn/defs
+
+* 15 bug fixes:
+
+  * Cleaned a bunch of old hacks. Initializing RubyLexer w/ Parser is cleaner now.
+  * Corrected some lex_state errors in process_token_keyword.
+  * Fixed ancient ruby2 change (use #lines) in ruby_parse_extract_error.
+  * Fixed bug where else without rescue only raises on 2.6+
+  * Fixed caller for getch and scan when DEBUG=1
+  * Fixed comments in the middle of message cascades.
+  * Fixed differences w/ symbol productions against ruby 2.7.
+  * Fixed dsym to use string_contents production.
+  * Fixed error in bdot2/3 in some edge cases. Fixed p_alt line.
+  * Fixed heredoc dedenting in the presence of empty lines. (mvz)
+  * Fixed some leading whitespace / comment processing
+  * Fixed up how class/module/defn/defs comments were collected.
+  * Overhauled ripper.rb to deal with buggy ripper w/ yydebug.
+  * Removed dsym from literal.
+  * Removed tUBANG lexeme but kept it distinct as a method name (eg: `def !@`).
+
+=== 3.17.0 / 2021-08-03
+
+* 1 minor enhancement:
+
+  * Added support for arg forwarding (eg `def f(...); m(...); end`) (presidentbeef)
+
+=== 3.16.0 / 2021-05-15
+
+* 1 major enhancement:
+
+  * Added tentative 3.0 support.
+
+* 3 minor enhancements:
+
+  * Added lexing for "beginless range" (bdots).
+  * Added parsing for bdots.
+  * Updated rake compare task to download xz files, bumped versions, etc
+
+* 4 bug fixes:
+
+  * Bump rake dependency to >= 10, < 15. (presidentbeef)
+  * Bump sexp_processor dependency to 4.15.1+. (pravi)
+  * Fixed minor state mismatch at the end of parsing to make diffing a little cleaner.
+  * Fixed normalizer to deal with new bison token syntax
+
+=== 3.15.1 / 2021-01-10
+
+* 1 bug fix:
+
+  * Bumped ruby version to include < 4 (trunk).
+
+=== 3.15.0 / 2020-08-31
+
+* 1 major enhancement:
+
+  * Added tentative 2.7 support.
+
+* 1 minor enhancement:
+
+  * Improved ruby_parse_extract_error's handling of moving slow files out.
+
+* 22 bug fixes:
+
+  * Bumped ruby version to include 3.0 (trunk).
+  * Fix an error related to empty ensure bodies. (presidentbeef)
+  * Fix handling of bad magic encoding comment.
+  * Fixed SystemStackError when parsing a huoooge hash, caused by a splat arg.
+  * Fixed a number of errors parsing do blocks in strange edge cases.
+  * Fixed a string backslash lexing bug when the string is an invalid encoding. (nijikon, gmcgibbon)
+  * Fixed bug assigning line number to some arg nodes.
+  * Fixed bug concatinating string literals with differing encodings.
+  * Fixed bug lexing heredoc w/ nasty mix of \r\n and \n.
+  * Fixed bug lexing multiple codepoints in \u{0000 1111 2222} forms.
+  * Fixed bug setting line numbers in empty xstrings in some contexts.
+  * Fixed edge case on call w/ begin + do block as an arg.
+  * Fixed handling of UTF BOM.
+  * Fixed handling of lexer state across string interpolation braces.
+  * Fixed infinite loop when lexing backslash+cr+newline (aka dos-files)
+  * Fixed lambda + do block edge case.
+  * Fixed lexing of some ?\M... and ?\C... edge cases.
+  * Fixed more do/brace block edge case failures.
+  * Fixed parsing bug where splat was used in the middle of a list.
+  * Fixed parsing of interpolation in heredoc-like strings. (presidentbeef)
+  * Fixed parsing some esoteric edge cases in op_asgn.
+  * Fixed unicode processing in ident chars so now they better mix.
+
+=== 3.14.2 / 2020-02-06
+
+* 1 minor enhancement:
+
+  * Cleaned up call_args and removed arg_blk_pass from ruby_parser_extras.rb! Yay!
+
+=== 3.14.1 / 2019-10-29
+
+* 1 minor enhancement:
+
+  * Declared that ruby_parser supports ruby 2.2 and up.
+
+* 3 bug fixes:
+
+  * Fixed a problem with %W with a null-byte terminator. (wtf?) (spohlenz)
+  * Fixed line numbering for command (eg methods without parentheses) arguments. (mvz)
+  * Fixed lineno on new dxstrs. (presidentbeef)
+
+=== 3.14.0 / 2019-09-24
+
+* 8 minor enhancements:
+
+  * Added Sexp#line_min & new line number debugging feature.
+  * Allow bin/ruby_parse to use RP_TIMEOUT to tweak default timeout for large files & slow debugging modes.
+  * Did you know that custom inspect can't be > 65 chars? 65! Me neither. Fixed.
+  * For now, value_expr == remove_begin.
+  * Improved error messages for unterminated regexps.
+  * Moved all STR_* consts into State::Values and made them State instances.
+  * Overhauled RubyLexer::State + Values to allow for completely separate groups of States.
+  * RubyParserExtras#remove_begin removes nested begins and patches line numbers.
+
+* 22 bug fixes:
+
+  * Changed EOF signaling to return [false, false] per doco.
+  * Changed RubyParserStuff#remove_begin to keep inner line number, if any. (mvz)
+  * Differentiated between lambda call and stabby proc. Added :lambda sexp. (whitequark)
+  * Extended State#== to check equal? first as an optimization.
+  * Fixed a bug in tab indent width.
+  * Fixed a line numbering bug for assignables.
+  * Fixed a line numbering bug for defns w/ nil bodies.
+  * Fixed another missing line number, driven out by ruby_to_ruby bootstrap.
+  * Fixed dedenting squiggle heredocs if interpolation is in the mix. (mvz)
+  * Fixed differentiation of `{ |a| ... }` vs `{ |a,| ... }`. Fixes #236 (whitequark).
+  * Fixed lex_state for symbols.
+  * Fixed lex_state for tSTRING_END and tREGEXP_END.
+  * Fixed line numbers for BEGIN block. (mvz)
+  * Fixed op_asgn1 w/ command_call having array instead of arglist. (mvz)
+  * Fixed parsing of () in a command-call (eg p).
+  * Fixed remaining missing line numbers by forcing extra checks during test and blowing up. They may still be wrong (esp after heredocs, ugh), but they're there! Special thank you to mvz for pushing on this and providing tests and PRs.
+  * Fixed some lex_state versioning issues with closing braces/bracket on ruby<25.
+  * Keep carriage return escapes in heredocs. (mvz)
+  * Massive overhaul of line number code.
+  * More line number fixes and extra tests from mvz. Thank you!
+  * Parsing BEGIN block when not at top level should raise an error. (mvz)
+  * Removed irb task from Rakefile as it has been pushed up to Hoe and doubles up here.
+
+=== 3.13.1 / 2019-03-25
+
+* 1 major enhancement:
+
+  * Overhauled RubyLexer#lex_state to allow for multi-state value.
+
+* 5 minor enhancements:
+
+  * Added RubyParser#current so scripts can poke around inside.
+  * Cleaned up and modernized bin/ruby_parse_extract_error.
+  * Removed RubyLexer#in_lex_state? in favor of State#=~ and #==.
+  * Renamed RubyLexer#in_arg_state? to is_after_operator?.
+  * Replaced all use of :expr_<*> symbols w/ bitfields.
+
+* 15 bug fixes:
+
+  * Added tests to show #233 already fixed.
+  * Fixed a lexer state bug dealing with complex edge case in hash  key w/ interpolated symbol and an array value. Not a great fix. (presidentbeef)
+  * Fixed a number of lex_state check problems.
+  * Fixed alias and undef keyword data.
+  * Fixed bug normalizing to utf-8 on tail of dstr (post-interpolation). (mvz)
+  * Fixed handling of hex chars for bare unicode escape. (mvz)
+  * Fixed handling of invalid string octals more gracefully. Needs generalization.
+  * Fixed lexing of regexps w/ backslashed unicode. I hope.
+  * Fixed parse_percent for EXPR_FITEM edge cases. Fixed bug #125.
+  * Fixed parsing of op_asgn on constants (eg ::X ||= 1).
+  * Fixed precedence of rescue modifier on assignments. (mvz/whitequark)
+  * Fixed some EXPR_FNAME edge cases.
+  * Fixed using unicode in gvars. Fixed bug 202. (331299)
+  * Handle invalid string literals more gracefully.
+  * Minor fix lexing %s[]
+
+=== 3.13.0 / 2019-03-12
+
+* 3 major enhancements:
+
+  * Removed 1.8 and 1.9 support. Moved to ruby_parser-legacy gem.
+  * Added tentative 2.6 support.
+  * Updated ruby_parser.yy to ruby 2.4 architecture.
+
+* 22 minor enhancements:
+
+  * Added debug3 and cmp3 rake tasks that use the above tools.
+  * Added tLONELY to compare/normalize.rb
+  * Added tools/munge.rb, tools/ripper.rb, and debugging.md
+  * 2.6: Added trailing .. and ... support.
+  * Extended StackState to log more state changes, making debugging easier.
+  * Extended StackState#store to take an initial value.
+  * Improved logging / debugging in StackState.
+  * Improved normalization and parser compare tasks.
+  * Improved tools/munge.rb output.
+  * In f_arglist, track in_kwarg if no parens.
+  * In process_newline_or_comment, handle NL if in_kwarg there vs normal.
+  * Refactored normalized values to WORDLIST.
+  * Refactored parser: push up to relop and rel_expr.
+  * Removed Keyword.keyword18.
+  * Removed RubyLexer version attr_accessors.
+  * Removed long dead deprecations internal to RubyParserStuff.
+  * Removed version arg to RubyLexer#initialize (default nil + no-op).
+  * Renamed Keyword.keyword19 to Keyword.keyword.
+  * Renamed process_bracing to process_brace_close
+  * Renamed process_curly_brace to process_brace_open
+  * Report first parse error message if all parser versions fail, not last.
+  * Updated parser to track against 2.6.
+
+* 11 bug fixes:
+
+  * Fix some shift/reduce errors.
+  * Fixed BEGIN blocks having different arg value that END blocks. (mvz)
+  * Fixed all reported unused non-terminals/rules.
+  * Fixed bug 272.
+  * Fixed bug in interpolated symbol lists. (strviola)
+  * Fixed bug where block shadow arguments were not registered as lvars. (akimd)
+  * Fixed bug where kwsplat args weren't treated as lvars. (mvz)
+  * Fixed lex_state and other internals in many cases.
+  * Fixed shebang in compare/normalize.rb to use env. Avoids rubygems freaking.
+  * Fixed some more internal state bugs.
+  * Fixed tRCURLY -> tSTRING_DEND for interpolation brace closes.
+
 === 3.12.0 / 2018-12-04
 
 * 3 minor enhancements:

data/Manifest.txt

@@ -6,13 +6,11 @@ Rakefile
 bin/ruby_parse
 bin/ruby_parse_extract_error
 compare/normalize.rb
+debugging.md
+gauntlet.md
 lib/.document
 lib/rp_extensions.rb
 lib/rp_stringscanner.rb
-lib/ruby18_parser.rb
-lib/ruby18_parser.y
-lib/ruby19_parser.rb
-lib/ruby19_parser.y
 lib/ruby20_parser.rb
 lib/ruby20_parser.y
 lib/ruby21_parser.rb
@@ -25,12 +23,22 @@ lib/ruby24_parser.rb
 lib/ruby24_parser.y
 lib/ruby25_parser.rb
 lib/ruby25_parser.y
+lib/ruby26_parser.rb
+lib/ruby26_parser.y
+lib/ruby27_parser.rb
+lib/ruby27_parser.y
+lib/ruby30_parser.rb
+lib/ruby30_parser.y
+lib/ruby3_parser.yy
 lib/ruby_lexer.rb
 lib/ruby_lexer.rex
 lib/ruby_lexer.rex.rb
+lib/ruby_lexer_strings.rb
 lib/ruby_parser.rb
 lib/ruby_parser.yy
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb
 test/test_ruby_parser_extras.rb
+tools/munge.rb
+tools/ripper.rb

data/README.rdoc

@@ -7,9 +7,9 @@ rdoc :: http://docs.seattlerb.org/ruby_parser
 == DESCRIPTION:
 
 ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-racc--which does by default use a C extension). RP's output is
-the same as ParseTree's output: s-expressions using ruby's arrays and
-base types.
+racc--which does by default use a C extension). It outputs
+s-expressions which can be manipulated and converted back to ruby via
+the ruby2ruby gem.
 
 As an example:
 
@@ -32,6 +32,7 @@ Tested against 801,039 files from the latest of all rubygems (as of 2013-05):
 * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
 * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
 * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+* 2.6 parser is at 99.9972% accuracy, 4.191 sigma
 
 == FEATURES/PROBLEMS: