ruby_parser 3.15.0 → 3.19.2

data/tools/munge.rb

@@ -76,12 +76,31 @@ def munge s
 
              # other
 
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
              'tLBRACK2',        "tLBRACK", # HACK
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
              "/* empty */",     "none",
-             /^\s*$/,           "none",
+             /^\s*$/,           "",
 
              "keyword_BEGIN",   "klBEGIN",
              "keyword_END",     "klEND",
@@ -89,6 +108,7 @@ def munge s
              /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
              /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
              "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
 
              # 2.6 collapses klBEGIN to kBEGIN
              "klBEGIN",   "kBEGIN",
@@ -112,9 +132,11 @@ def munge s
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
 
              /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
              /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
              /\$?@(\d+) */,         "", # TODO: remove?
@@ -130,7 +152,11 @@ def munge s
     end
   end
 
-  s.strip.squeeze " "
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
 end
 
 ARGF.each_line do |line|
@@ -144,19 +170,19 @@ ARGF.each_line do |line|
   when /^Reading a token: Next token is token (.*?) \(\)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Reading a token: / then
     next # skip
   when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Next token is token ("[^"]+"|\S+)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^read\s+false/ then # read    false($end) "$end"
     puts "next token is EOF"
@@ -164,6 +190,8 @@ ARGF.each_line do |line|
     # do nothing
   when /^.:scan=>\["([^"]+)"/ then
     puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
   when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
     reduce_line = $2.to_i
   when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
@@ -172,7 +200,7 @@ ARGF.each_line do |line|
   when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
     stack << "none" if stack.empty?
     item = munge $1
-    x = stack.map { |s| s.strip }.join " "
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
     if x != item then # prevent kdef -> kdef
       if $v && reduce_line then
         puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "
@@ -197,8 +225,8 @@ ARGF.each_line do |line|
     puts line.gsub("true", "1").gsub("false", "0")
   when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
     a, b, c = $1.upcase, $2.upcase, $3
-    a.gsub! /EXPR_/, ""
-    b.gsub! /EXPR_/, ""
+    a.gsub!(/EXPR_/, "")
+    b.gsub!(/EXPR_/, "")
     if c && $v then
       puts "lex_state: #{a} -> #{b} at #{c}"
     else

data/tools/ripper.rb

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby -ws
 
-$d ||= false
-$p ||= false
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
 
 require "ripper/sexp"
 require "pp" if $p
@@ -19,15 +20,19 @@ end
 
 ARGV.each do |path|
   src = path == "-" ? $stdin.read : File.read(path)
-  rip = MySexpBuilder.new src
-  rip.yydebug = $d
 
-  sexp = rip.parse
-
-  if rip.error? then
-    warn "skipping"
-    next
-  end
+  sexp = if $b then
+           Ripper.sexp src
+         else
+           rip = MySexpBuilder.new src
+           rip.yydebug = $d
+           rip.parse
+
+           if rip.error? then
+             warn "skipping"
+             next
+           end
+         end
 
   puts "accept"
 

metadata

@@ -1,18 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.15.0
+  version: 3.19.2
 platform: ruby
 authors:
 - Ryan Davis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBDANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBjANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE5MTIxMzAwMDIwNFoXDTIwMTIxMjAwMDIwNFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIxMTIyMzIzMTkwNFoXDTIyMTIyMzIzMTkwNFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQCkkcHqAa6IKLYGl93rn78J3L+LnqyxaA059n4IGMHWN5bv9KBQnIjOrpLadtYZ
-  vhWkunWDKdfVapBEq5+T4HzqnsEXC3aCv6JEKJY6Zw7iSzl0M8hozuzRr+w46wvT
-  fV2yTN6QTVxqbMsJJyjosks4ZdQYov2zdvQpt1HsLi+Qmckmg8SPZsd+T8uiiBCf
-  b+1ORSM5eEfBQenPXy83LZcoQz8i6zVB4aAfTGGdhxjoMGUEmSZ6xpkOzmnGa9QK
-  m5x9IDiApM+vCELNwDXXGNFEnQBBK+wAe4Pek8o1V1TTOxL1kGPewVOitX1p3xoN
-  h7iEjga8iM1LbZUfiISZ+WrB
+  AQCKB5jfsuSnKb+t/Wrh3UpdkmX7TrEsjVmERC0pPqzQ5GQJgmEXDD7oMgaKXaAq
+  x2m+KSZDrqk7c8uho5OX6YMqg4KdxehfSLqqTZGoeV78qwf/jpPQZKTf+W9gUSJh
+  zsWpo4K50MP+QtdSbKXZwjAafpQ8hK0MnnZ/aeCsW9ov5vdXpYbf3dpg6ADXRGE7
+  lQY2y1tJ5/chqu6h7dQmnm2ABUqx9O+JcN9hbCYoA5i/EeubUEtFIh2w3SpO6YfB
+  JFmxn4h9YO/pVdB962BdBNNDia0kgIjI3ENnkLq0dKpYU3+F3KhEuTksLO0L6X/V
+  YsuyUzsMz6GQA4khyaMgKNSD
   -----END CERTIFICATE-----
-date: 2020-09-01 00:00:00.000000000 Z
+date: 2022-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -37,42 +37,62 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
 - !ruby/object:Gem::Dependency
   name: oedipus_lex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -93,34 +113,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '7'
-- !ruby/object:Gem::Dependency
-  name: racc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.25'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.25'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). It outputs
@@ -148,6 +154,10 @@ description: |-
   * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+  * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
+  * 3.0 parser has a 100% parse rate.
+    * Tested against 2,672,412 unique ruby files across 167k gems.
+    * As do all the others now, basically.
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -159,6 +169,7 @@ extra_rdoc_files:
 - Manifest.txt
 - README.rdoc
 - debugging.md
+- gauntlet.md
 files:
 - ".autotest"
 - History.rdoc
@@ -169,6 +180,7 @@ files:
 - bin/ruby_parse_extract_error
 - compare/normalize.rb
 - debugging.md
+- gauntlet.md
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
@@ -188,9 +200,15 @@ files:
 - lib/ruby26_parser.y
 - lib/ruby27_parser.rb
 - lib/ruby27_parser.y
+- lib/ruby30_parser.rb
+- lib/ruby30_parser.y
+- lib/ruby31_parser.rb
+- lib/ruby31_parser.y
+- lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
+- lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
 - lib/ruby_parser.yy
 - lib/ruby_parser_extras.rb
@@ -205,7 +223,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/seattlerb/ruby_parser
   bug_tracker_uri: https://github.com/seattlerb/ruby_parser/issues
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.rdoc
@@ -218,15 +236,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
   - - "<"
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.3.12
+signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which
   does by default use a C extension)