ruby_parser 3.15.1 → 3.18.1

data/tools/munge.rb

@@ -76,12 +76,31 @@ def munge s
 
              # other
 
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
              'tLBRACK2',        "tLBRACK", # HACK
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
              "/* empty */",     "none",
-             /^\s*$/,           "none",
+             /^\s*$/,           "",
 
              "keyword_BEGIN",   "klBEGIN",
              "keyword_END",     "klEND",
@@ -89,6 +108,7 @@ def munge s
              /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
              /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
              "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
 
              # 2.6 collapses klBEGIN to kBEGIN
              "klBEGIN",   "kBEGIN",
@@ -112,9 +132,11 @@ def munge s
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
 
              /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
              /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
              /\$?@(\d+) */,         "", # TODO: remove?
@@ -130,7 +152,11 @@ def munge s
     end
   end
 
-  s.strip.squeeze " "
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
 end
 
 ARGF.each_line do |line|
@@ -144,19 +170,19 @@ ARGF.each_line do |line|
   when /^Reading a token: Next token is token (.*?) \(\)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Reading a token: / then
     next # skip
   when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Next token is token ("[^"]+"|\S+)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^read\s+false/ then # read    false($end) "$end"
     puts "next token is EOF"
@@ -164,6 +190,8 @@ ARGF.each_line do |line|
     # do nothing
   when /^.:scan=>\["([^"]+)"/ then
     puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
   when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
     reduce_line = $2.to_i
   when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
@@ -172,7 +200,7 @@ ARGF.each_line do |line|
   when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
     stack << "none" if stack.empty?
     item = munge $1
-    x = stack.map { |s| s.strip }.join " "
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
     if x != item then # prevent kdef -> kdef
       if $v && reduce_line then
         puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "
@@ -197,8 +225,8 @@ ARGF.each_line do |line|
     puts line.gsub("true", "1").gsub("false", "0")
   when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
     a, b, c = $1.upcase, $2.upcase, $3
-    a.gsub! /EXPR_/, ""
-    b.gsub! /EXPR_/, ""
+    a.gsub!(/EXPR_/, "")
+    b.gsub!(/EXPR_/, "")
     if c && $v then
       puts "lex_state: #{a} -> #{b} at #{c}"
     else

data/tools/ripper.rb

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby -ws
 
-$d ||= false
-$p ||= false
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
 
 require "ripper/sexp"
 require "pp" if $p
@@ -19,15 +20,19 @@ end
 
 ARGV.each do |path|
   src = path == "-" ? $stdin.read : File.read(path)
-  rip = MySexpBuilder.new src
-  rip.yydebug = $d
 
-  sexp = rip.parse
-
-  if rip.error? then
-    warn "skipping"
-    next
-  end
+  sexp = if $b then
+           Ripper.sexp src
+         else
+           rip = MySexpBuilder.new src
+           rip.yydebug = $d
+           rip.parse
+
+           if rip.error? then
+             warn "skipping"
+             next
+           end
+         end
 
   puts "accept"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.15.1
+  version: 3.18.1
 platform: ruby
 authors:
 - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
   d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
   KToW560QIey7SPfHWduzFJnV
   -----END CERTIFICATE-----
-date: 2021-01-11 00:00:00.000000000 Z
+date: 2021-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -37,42 +37,62 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.9'
+        version: '4.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '15'
 - !ruby/object:Gem::Dependency
   name: oedipus_lex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -93,20 +113,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '7'
-- !ruby/object:Gem::Dependency
-  name: racc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
@@ -148,6 +154,7 @@ description: |-
   * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+  * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -159,6 +166,7 @@ extra_rdoc_files:
 - Manifest.txt
 - README.rdoc
 - debugging.md
+- gauntlet.md
 files:
 - ".autotest"
 - History.rdoc
@@ -169,6 +177,7 @@ files:
 - bin/ruby_parse_extract_error
 - compare/normalize.rb
 - debugging.md
+- gauntlet.md
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
@@ -188,9 +197,13 @@ files:
 - lib/ruby26_parser.y
 - lib/ruby27_parser.rb
 - lib/ruby27_parser.y
+- lib/ruby30_parser.rb
+- lib/ruby30_parser.y
+- lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
+- lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
 - lib/ruby_parser.yy
 - lib/ruby_parser_extras.rb
@@ -225,7 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.16
 signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which