ruby_parser 3.13.1 → 3.21.0

data/test/test_ruby_parser_extras.rb

@@ -16,11 +16,11 @@ class TestStackState < Minitest::Test
     p = RubyParser.latest
     s = nil
 
-    out, err = capture_io do
+    out, err = capture_io {
       s = p.handle_encoding str
-    end
+    }
 
-    assert_equal orig_str.sub(/\357\273\277/, ''), s
+    assert_equal orig_str.sub(/\357\273\277/, ""), s
 
     exp_err = ""
 
@@ -162,7 +162,7 @@ end
 
 class TestEnvironment < Minitest::Test
   def deny t
-    assert ! t
+    assert !t
   end
 
   def setup
@@ -224,3 +224,38 @@ class TestEnvironment < Minitest::Test
     assert_equal expected0, @env.all
   end
 end
+
+class Fake20
+  include RubyParserStuff
+
+  def initialize
+  end
+
+  def s(*a) # bypass lexer/lineno stuff that RP overrides in
+    Kernel.send :s, *a
+  end
+end
+
+class TestValueExpr < Minitest::Test
+  def assert_value_expr exp, input
+    assert_equal exp, Fake20.new.value_expr(input.line(1))
+  end
+
+  def assert_remove_begin exp, input
+    assert_equal exp, Fake20.new.remove_begin(input.line(1))
+  end
+
+  def test_value_expr
+    assert_value_expr s(:nil),                     s(:begin)
+    assert_value_expr s(:nil),                     s(:begin, s(:nil))
+    assert_value_expr s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_value_expr s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+
+  def test_remove_begin
+    assert_remove_begin s(:nil),                     s(:begin)
+    assert_remove_begin s(:nil),                     s(:begin, s(:nil))
+    assert_remove_begin s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_remove_begin s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+end

data/tools/munge.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby -ws
+#!/usr/bin/env ruby -ws
 
 $v ||= false
 
@@ -76,12 +76,31 @@ def munge s
 
              # other
 
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
              'tLBRACK2',        "tLBRACK", # HACK
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
              "/* empty */",     "none",
-             /^\s*$/,           "none",
+             /^\s*$/,           "",
 
              "keyword_BEGIN",   "klBEGIN",
              "keyword_END",     "klEND",
@@ -89,6 +108,7 @@ def munge s
              /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
              /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
              "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
 
              # 2.6 collapses klBEGIN to kBEGIN
              "klBEGIN",   "kBEGIN",
@@ -112,12 +132,16 @@ def munge s
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
 
              /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
              /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
              /\$?@(\d+) */,         "", # TODO: remove?
+
+             /_EXPR/,               "",
             ]
 
   renames.each_slice(2) do |(a, b)|
@@ -128,7 +152,11 @@ def munge s
     end
   end
 
-  s.strip.squeeze " "
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
 end
 
 ARGF.each_line do |line|
@@ -142,19 +170,23 @@ ARGF.each_line do |line|
   when /^Reading a token: Next token is token (.*?) \(\)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Reading a token: / then
     next # skip
+  when /^Reading a token$/ then # wtf?
+    next # skip
+  when /^(?:add_delayed_token|parser_dispatch)/ then # dunno what this is yet
+    next # skip
   when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
-  when /^Next token is token (\S+)/ then
+  when /^Next token is token ("[^"]+"|\S+)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^read\s+false/ then # read    false($end) "$end"
     puts "next token is EOF"
@@ -162,6 +194,8 @@ ARGF.each_line do |line|
     # do nothing
   when /^.:scan=>\["([^"]+)"/ then
     puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
   when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
     reduce_line = $2.to_i
   when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
@@ -170,7 +204,7 @@ ARGF.each_line do |line|
   when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
     stack << "none" if stack.empty?
     item = munge $1
-    x = stack.map { |s| s.strip }.join " "
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
     if x != item then # prevent kdef -> kdef
       if $v && reduce_line then
         puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "
@@ -182,7 +216,9 @@ ARGF.each_line do |line|
     reduce_line = nil
     stack.clear
   when /^reduce/ then # ruby_parser side
-    puts munge line.chomp
+    s = munge line.chomp
+    next if s =~ /reduce\s+(\w+) --> \1/
+    puts s
     puts
   when /^(\w+_stack)\.(\w+)/ then
     # TODO: make pretty, but still informative w/ line numbers etc
@@ -193,11 +229,14 @@ ARGF.each_line do |line|
     # puts line
     # TODO: make pretty, but still informative w/ line numbers etc
     puts line.gsub("true", "1").gsub("false", "0")
-  when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
-    if $3 && $v then
-      puts "lex_state: #{$1.upcase} -> #{$2.upcase} at #{$3}"
+  when /^lex_state: :?([\w|()]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
+    a, b, c = $1.upcase, $2.upcase, $3
+    a.gsub!(/EXPR_/, "")
+    b.gsub!(/EXPR_/, "")
+    if c && $v then
+      puts "lex_state: #{a} -> #{b} at #{c}"
     else
-      puts "lex_state: #{$1.upcase} -> #{$2.upcase}"
+      puts "lex_state: #{a} -> #{b}"
     end
   when /debug|FUCK/ then
     puts line.chomp

data/tools/ripper.rb

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby -ws
 
-$d ||= false
-$p ||= false
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
 
 require "ripper/sexp"
 require "pp" if $p
@@ -11,14 +12,31 @@ if ARGV.empty? then
   ARGV << "-"
 end
 
+class MySexpBuilder < Ripper::SexpBuilderPP
+  def on_parse_error msg
+    Kernel.warn msg
+  end
+end
+
 ARGV.each do |path|
   src = path == "-" ? $stdin.read : File.read(path)
-  rip = Ripper::SexpBuilderPP.new src
-  rip.yydebug = $d
 
-  sexp = rip.parse
+  sexp = nil
+
+  if $b then
+    sexp = Ripper.sexp src
+  else
+    rip = MySexpBuilder.new src
+    rip.yydebug = $d
+    sexp = rip.parse
+
+    if rip.error? then
+      warn "skipping"
+      next
+    end
+  end
 
-  puts "accept" unless rip.error?
+  puts "accept"
 
   if $p then
     pp sexp

metadata

@@ -1,18 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.13.1
+  version: 3.21.0
 platform: ruby
 authors:
 - Ryan Davis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBAzANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBCDANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE4MTIwNDIxMzAxNFoXDTE5MTIwNDIxMzAxNFowRTETMBEGA1UE
+  GRYDY29tMB4XDTI0MDEwMjIxMjEyM1oXDTI1MDEwMTIxMjEyM1owRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQCbJwLmpJR2PomLU+Zzw3KRzH/hbyUWc/ftru71AopZ1fy4iY9J/BW5QYKVYwbP
-  V0FSBWtvfI/RdwfKGtuGhPKECZgmLieGuZ3XCc09qPu1bdg7i/tu1p0t0c6163ku
-  nDMDIC/t/DAFK0TY9I3HswuyZGbLW7rgF0DmiuZdN/RPhHq2pOLMLXJmFclCb/im
-  9yToml/06TJdUJ5p64mkBs0TzaK66DIB1Smd3PdtfZqoRV+EwaXMdx0Hb3zdR1JR
-  Em82dBUFsipwMLCYj39kcyHWAxyl6Ae1Cn9r/ItVBCxoeFdrHjfavnrIEoXUt4bU
-  UfBugfLD19bu3nvL+zTAGx/U
+  AQCygvpmncmkiSs9r/Kceo4bBPDszhTv6iBi4LwMReqnFrpNLMOWJw7xi8x+3eL2
+  XS09ZPNOt2zm70KmFouBMgOysnDY4k2dE8uF6B8JbZOO8QfalW+CoNBliefOTcn2
+  bg5IOP7UoGM5lC174/cbDJrJnRG9bzig5FAP0mvsgA8zgTRXQzIUAZEo92D5K7p4
+  B4/O998ho6BSOgYBI9Yk1ttdCtti6Y+8N9+fZESsjtWMykA+WXWeGUScHqiU+gH8
+  S7043fq9EbQdBr2AXdj92+CDwuTfHI6/Hj5FVBDULufrJaan4xUgL70Hvc6pTTeW
+  deKfBjgVAq7EYHu1AczzlUly
   -----END CERTIFICATE-----
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2024-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -37,42 +37,62 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
 - !ruby/object:Gem::Dependency
   name: oedipus_lex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -93,39 +113,25 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '7'
-- !ruby/object:Gem::Dependency
-  name: racc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.17'
+        version: '4.2'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-  racc--which does by default use a C extension). RP's output is
-  the same as ParseTree's output: s-expressions using ruby's arrays and
-  base types.
+  racc--which does by default use a C extension). It outputs
+  s-expressions which can be manipulated and converted back to ruby via
+  the ruby2ruby gem.
 
   As an example:
 
@@ -148,6 +154,10 @@ description: |-
   * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+  * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
+  * 3.0 parser has a 100% parse rate.
+    * Tested against 2,672,412 unique ruby files across 167k gems.
+    * As do all the others now, basically.
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -159,6 +169,7 @@ extra_rdoc_files:
 - Manifest.txt
 - README.rdoc
 - debugging.md
+- gauntlet.md
 files:
 - ".autotest"
 - History.rdoc
@@ -169,28 +180,29 @@ files:
 - bin/ruby_parse_extract_error
 - compare/normalize.rb
 - debugging.md
+- gauntlet.md
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
-- lib/ruby20_parser.rb
-- lib/ruby20_parser.y
-- lib/ruby21_parser.rb
-- lib/ruby21_parser.y
-- lib/ruby22_parser.rb
-- lib/ruby22_parser.y
-- lib/ruby23_parser.rb
-- lib/ruby23_parser.y
-- lib/ruby24_parser.rb
-- lib/ruby24_parser.y
-- lib/ruby25_parser.rb
-- lib/ruby25_parser.y
-- lib/ruby26_parser.rb
-- lib/ruby26_parser.y
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
+- lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
-- lib/ruby_parser.yy
+- lib/ruby_parser2.yy
+- lib/ruby_parser20.rb
+- lib/ruby_parser21.rb
+- lib/ruby_parser22.rb
+- lib/ruby_parser23.rb
+- lib/ruby_parser24.rb
+- lib/ruby_parser25.rb
+- lib/ruby_parser26.rb
+- lib/ruby_parser27.rb
+- lib/ruby_parser3.yy
+- lib/ruby_parser30.rb
+- lib/ruby_parser31.rb
+- lib/ruby_parser32.rb
+- lib/ruby_parser33.rb
 - lib/ruby_parser_extras.rb
 - test/test_ruby_lexer.rb
 - test/test_ruby_parser.rb
@@ -200,8 +212,10 @@ files:
 homepage: https://github.com/seattlerb/ruby_parser
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/seattlerb/ruby_parser
+  bug_tracker_uri: https://github.com/seattlerb/ruby_parser/issues
+post_install_message:
 rdoc_options:
 - "--main"
 - README.rdoc
@@ -211,15 +225,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
-signing_key: 
+rubygems_version: 3.5.3
+signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which
   does by default use a C extension)