ruby_parser 3.12.0 → 3.18.1

data/test/test_ruby_parser_extras.rb

@@ -16,11 +16,11 @@ class TestStackState < Minitest::Test
     p = RubyParser.latest
     s = nil
 
-    out, err = capture_io do
+    out, err = capture_io {
       s = p.handle_encoding str
-    end
+    }
 
-    assert_equal orig_str.sub(/\357\273\277/, ''), s
+    assert_equal orig_str.sub(/\357\273\277/, ""), s
 
     exp_err = ""
 
@@ -162,7 +162,7 @@ end
 
 class TestEnvironment < Minitest::Test
   def deny t
-    assert ! t
+    assert !t
   end
 
   def setup
@@ -224,3 +224,38 @@ class TestEnvironment < Minitest::Test
     assert_equal expected0, @env.all
   end
 end
+
+class Fake20
+  include RubyParserStuff
+
+  def initialize
+  end
+
+  def s(*a) # bypass lexer/lineno stuff that RP overrides in
+    Kernel.send :s, *a
+  end
+end
+
+class TestValueExpr < Minitest::Test
+  def assert_value_expr exp, input
+    assert_equal exp, Fake20.new.value_expr(input.line(1))
+  end
+
+  def assert_remove_begin exp, input
+    assert_equal exp, Fake20.new.remove_begin(input.line(1))
+  end
+
+  def test_value_expr
+    assert_value_expr s(:nil),                     s(:begin)
+    assert_value_expr s(:nil),                     s(:begin, s(:nil))
+    assert_value_expr s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_value_expr s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+
+  def test_remove_begin
+    assert_remove_begin s(:nil),                     s(:begin)
+    assert_remove_begin s(:nil),                     s(:begin, s(:nil))
+    assert_remove_begin s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_remove_begin s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+end

data/tools/munge.rb

@@ -0,0 +1,250 @@
+#!/usr/bin/env ruby -ws
+
+$v ||= false
+
+stack = []
+last_token = nil
+reduce_line = nil
+
+def munge s
+  renames = [
+             "'='",             "tEQL",
+             "'!'",             "tBANG",
+             "'%'",             "tPERCENT",
+             "'&'",             "tAMPER2",
+             "'('",             "tLPAREN2",
+             "')'",             "tRPAREN",
+             "'*'",             "tSTAR2",
+             "'+'",             "tPLUS",
+             "','",             "tCOMMA",
+             "'-'",             "tMINUS",
+             "'.'",             "tDOT",
+             "'/'",             "tDIVIDE",
+             "';'",             "tSEMI",
+             "':'",             "tCOLON",
+             "'<'",             "tLT",
+             "'>'",             "tGT",
+             "'?'",             "tEH",
+             "'['",             "tLBRACK",
+             "'\\n'",           "tNL",
+             "']'",             "tRBRACK",
+             "'^'",             "tCARET",
+             "'`'",             "tBACK_REF2",
+             "'{'",             "tLCURLY",
+             "'|'",             "tPIPE",
+             "'}'",             "tRCURLY",
+             "'~'",             "tTILDE",
+             '"["',             "tLBRACK",
+
+             # 2.0 changes?
+             '"<=>"',            "tCMP",
+             '"=="',             "tEQ",
+             '"==="',            "tEQQ",
+             '"!~"',             "tNMATCH",
+             '"=~"',             "tMATCH",
+             '">="',             "tGEQ",
+             '"<="',             "tLEQ",
+             '"!="',             "tNEQ",
+             '"<<"',             "tLSHFT",
+             '">>"',             "tRSHFT",
+             '"*"',              "tSTAR",
+
+             '".."',             "tDOT2",
+
+             '"&"',              "tAMPER",
+             '"&&"',             "tANDOP",
+             '"&."',             "tLONELY",
+             '"||"',             "tOROP",
+
+             '"..."',            "tDOT3",
+             '"**"',             "tPOW",
+             '"unary+"',         "tUPLUS",
+             '"unary-"',         "tUMINUS",
+             '"[]"',             "tAREF",
+             '"[]="',            "tASET",
+             '"::"',             "tCOLON2",
+             '"{ arg"',          "tLBRACE_ARG",
+             '"( arg"',          "tLPAREN_ARG",
+             '"("',              "tLPAREN",
+             'rparen',           "tRPAREN",
+             '"{"',              "tLBRACE",
+             '"=>"',             "tASSOC",
+             '"->"',             "tLAMBDA",
+             '":: at EXPR_BEG"', "tCOLON3",
+             '"**arg"',          "tDSTAR",
+             '","',              "tCOMMA",
+
+             # other
+
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
+             'tLBRACK2',        "tLBRACK", # HACK
+
+             "' '",             "tSPACE", # needs to be later to avoid bad hits
+
+             "/* empty */",     "none",
+             /^\s*$/,           "",
+
+             "keyword_BEGIN",   "klBEGIN",
+             "keyword_END",     "klEND",
+             /keyword_(\w+)/,   proc { "k#{$1.upcase}" },
+             /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
+             "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
+
+             # 2.6 collapses klBEGIN to kBEGIN
+             "klBEGIN",   "kBEGIN",
+             "klEND",     "kEND",
+
+             /keyword_(\w+)/,          proc { "k#{$1.upcase}" },
+             /\bk_([^_][a-z_]+)/,      proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,         proc { "k#{$1.upcase}_MOD" },
+
+             "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
+
+             # UGH
+             "k_LINE__",       "k__LINE__",
+             "k_FILE__",       "k__FILE__",
+             "k_ENCODING__",   "k__ENCODING__",
+
+             '"defined?"',     "kDEFINED",
+
+             "<none>",         "none",
+
+             '"do (for condition)"', "kDO_COND",
+             '"do (for lambda)"',    "kDO_LAMBDA",
+             '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
+
+             /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
+
+             /@(\d+)(\s+|$)/,       "",
+             /\$?@(\d+) */,         "", # TODO: remove?
+
+             /_EXPR/,               "",
+            ]
+
+  renames.each_slice(2) do |(a, b)|
+    if Proc === b then
+      s.gsub!(a, &b)
+    else
+      s.gsub!(a, b)
+    end
+  end
+
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
+end
+
+ARGF.each_line do |line|
+  case line
+  when /^(Stack now|Entering state|Shifting|Cleanup|Starting)/ then
+    # do nothing
+  when /^vtable_/ then
+    # do nothing
+  when /Gem::MissingSpecError/ then
+    # do nothing -- ruby 2.5 is being bitchy?
+  when /^Reading a token: Next token is token (.*?) \(\)/ then
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^Reading a token: / then
+    next # skip
+  when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^Next token is token ("[^"]+"|\S+)/ then
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^read\s+false/ then # read    false($end) "$end"
+    puts "next token is EOF"
+  when /^Now at end of input./ then
+    # do nothing
+  when /^.:scan=>\["([^"]+)"/ then
+    puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
+  when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
+    reduce_line = $2.to_i
+  when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
+    item = $1
+    stack << munge(item)
+  when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
+    stack << "none" if stack.empty?
+    item = munge $1
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
+    if x != item then # prevent kdef -> kdef
+      if $v && reduce_line then
+        puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "
+      else
+        puts "reduce #{x} --> #{item}".squeeze " "
+      end
+      puts
+    end
+    reduce_line = nil
+    stack.clear
+  when /^reduce/ then # ruby_parser side
+    puts munge line.chomp
+    puts
+  when /^(\w+_stack)\.(\w+)/ then
+    # TODO: make pretty, but still informative w/ line numbers etc
+    puts line.gsub("true", "1").gsub("false", "0")
+    # puts "#{$1}(#{$2})"
+  when /^(\w+_stack(\(\w+\))?: \S+)/ then
+    # _data = $v ? line.chomp : $1
+    # puts line
+    # TODO: make pretty, but still informative w/ line numbers etc
+    puts line.gsub("true", "1").gsub("false", "0")
+  when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
+    a, b, c = $1.upcase, $2.upcase, $3
+    a.gsub!(/EXPR_/, "")
+    b.gsub!(/EXPR_/, "")
+    if c && $v then
+      puts "lex_state: #{a} -> #{b} at #{c}"
+    else
+      puts "lex_state: #{a} -> #{b}"
+    end
+  when /debug|FUCK/ then
+    puts line.chomp
+  when /^(#.*parse error|on )/ then
+    puts line.chomp
+  when /^(goto|shift| +\[|$)/ then # racc
+    # do nothing
+  # when /^Reading a token: Now at end of input./ then
+  #   # puts "EOF"
+  # when /^Reading a token: Next token is token (.+)/ then
+  #   puts "READ: #{$1.inspect}"
+  when /^accept/ then
+    puts "DONE"
+  else
+    puts "unparsed: #{line.chomp}"
+  end
+end

data/tools/ripper.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby -ws
+
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
+
+require "ripper/sexp"
+require "pp" if $p
+
+if ARGV.empty? then
+  warn "reading from stdin"
+  ARGV << "-"
+end
+
+class MySexpBuilder < Ripper::SexpBuilderPP
+  def on_parse_error msg
+    Kernel.warn msg
+  end
+end
+
+ARGV.each do |path|
+  src = path == "-" ? $stdin.read : File.read(path)
+
+  sexp = if $b then
+           Ripper.sexp src
+         else
+           rip = MySexpBuilder.new src
+           rip.yydebug = $d
+           rip.parse
+
+           if rip.error? then
+             warn "skipping"
+             next
+           end
+         end
+
+  puts "accept"
+
+  if $p then
+    pp sexp
+  else
+    p sexp
+  end
+end

metadata

@@ -1,18 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.12.0
+  version: 3.18.1
 platform: ruby
 authors:
 - Ryan Davis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBAjANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBTANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE3MTEyMTIxMTExMFoXDTE4MTEyMTIxMTExMFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIwMTIyMjIwMzgzMFoXDTIxMTIyMjIwMzgzMFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -21,15 +21,15 @@ cert_chain:
   GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
-  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBBQUAA4IB
-  AQAfAXSQpsW7YSxd1csRtA/M4Zt0AMXFMd76GJ8Lgtg8G0+VFbdChRyDuDb0kPlW
-  h9QQX/YABfCW8vxmssbMGrP+VGBAn7BbdTcfTlgCWrvMX1uL5aRL74nA4urKXqdW
-  a0nP70K4958P3GffBdtE3KGkU5xstFnXGajxuBRnL66E15KU0BNehVxdG258bdPu
-  EKN6MqBPftFiev3tuwqDV11r2GquDpniYcT+Mi8/PgeAgVT/afBeVgbB3KaZeTRR
-  AhXhF6Wi2GTMezlj5jlI5XV7WsJUSwTp/YiVvcmT74ZaCRvexm6EnNhkrvJJ1Xeu
-  V+HB+LYYhXWitInO/eXxDrFB
+  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
+  AQAE3XRm1YZcCVjAJy5yMZvTOFrS7B2SYErc+0QwmKYbHztTTDY2m5Bii+jhpuxh
+  H+ETcU1z8TUKLpsBUP4kUpIRowkVN1p/jKapV8T3Rbwq+VuYFe+GMKsf8wGZSecG
+  oMQ8DzzauZfbvhe2kDg7G9BBPU0wLQlY25rDcCy9bLnD7R0UK3ONqpwvsI5I7x5X
+  ZIMXR0a9/DG+55mawwdGzCQobDKiSNLK89KK7OcNTALKU0DfgdTkktdgKchzKHqZ
+  d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
+  KToW560QIey7SPfHWduzFJnV
   -----END CERTIFICATE-----
-date: 2018-12-04 00:00:00.000000000 Z
+date: 2021-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -37,42 +37,62 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
 - !ruby/object:Gem::Dependency
   name: oedipus_lex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -93,39 +113,25 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '7'
-- !ruby/object:Gem::Dependency
-  name: racc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '3.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '3.22'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-  racc--which does by default use a C extension). RP's output is
-  the same as ParseTree's output: s-expressions using ruby's arrays and
-  base types.
+  racc--which does by default use a C extension). It outputs
+  s-expressions which can be manipulated and converted back to ruby via
+  the ruby2ruby gem.
 
   As an example:
 
@@ -148,6 +154,7 @@ description: |-
   * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+  * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -158,6 +165,8 @@ extra_rdoc_files:
 - History.rdoc
 - Manifest.txt
 - README.rdoc
+- debugging.md
+- gauntlet.md
 files:
 - ".autotest"
 - History.rdoc
@@ -167,13 +176,11 @@ files:
 - bin/ruby_parse
 - bin/ruby_parse_extract_error
 - compare/normalize.rb
+- debugging.md
+- gauntlet.md
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
-- lib/ruby18_parser.rb
-- lib/ruby18_parser.y
-- lib/ruby19_parser.rb
-- lib/ruby19_parser.y
 - lib/ruby20_parser.rb
 - lib/ruby20_parser.y
 - lib/ruby21_parser.rb
@@ -186,20 +193,32 @@ files:
 - lib/ruby24_parser.y
 - lib/ruby25_parser.rb
 - lib/ruby25_parser.y
+- lib/ruby26_parser.rb
+- lib/ruby26_parser.y
+- lib/ruby27_parser.rb
+- lib/ruby27_parser.y
+- lib/ruby30_parser.rb
+- lib/ruby30_parser.y
+- lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
+- lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
 - lib/ruby_parser.yy
 - lib/ruby_parser_extras.rb
 - test/test_ruby_lexer.rb
 - test/test_ruby_parser.rb
 - test/test_ruby_parser_extras.rb
+- tools/munge.rb
+- tools/ripper.rb
 homepage: https://github.com/seattlerb/ruby_parser
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/seattlerb/ruby_parser
+  bug_tracker_uri: https://github.com/seattlerb/ruby_parser/issues
+post_install_message:
 rdoc_options:
 - "--main"
 - README.rdoc
@@ -209,16 +228,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.2.16
+signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which
   does by default use a C extension)