ruby_parser 3.17.0 → 3.18.0

data/tools/munge.rb

@@ -76,12 +76,31 @@ def munge s
 
              # other
 
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
              'tLBRACK2',        "tLBRACK", # HACK
 
              "' '",             "tSPACE", # needs to be later to avoid bad hits
 
              "/* empty */",     "none",
-             /^\s*$/,           "none",
+             /^\s*$/,           "",
 
              "keyword_BEGIN",   "klBEGIN",
              "keyword_END",     "klEND",
@@ -89,6 +108,7 @@ def munge s
              /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
              /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
              "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
 
              # 2.6 collapses klBEGIN to kBEGIN
              "klBEGIN",   "kBEGIN",
@@ -112,9 +132,11 @@ def munge s
              '"do (for condition)"', "kDO_COND",
              '"do (for lambda)"',    "kDO_LAMBDA",
              '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
 
              /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
              /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
 
              /@(\d+)(\s+|$)/,       "",
              /\$?@(\d+) */,         "", # TODO: remove?
@@ -130,7 +152,11 @@ def munge s
     end
   end
 
-  s.strip.squeeze " "
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
 end
 
 ARGF.each_line do |line|
@@ -144,19 +170,19 @@ ARGF.each_line do |line|
   when /^Reading a token: Next token is token (.*?) \(\)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Reading a token: / then
     next # skip
   when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^Next token is token ("[^"]+"|\S+)/ then
     token = munge $1
     next if last_token == token
-    puts "next token is %p (%p)" % [token, last_token]
+    puts "next token is %p" % [token]
     last_token = token
   when /^read\s+false/ then # read    false($end) "$end"
     puts "next token is EOF"
@@ -164,6 +190,8 @@ ARGF.each_line do |line|
     # do nothing
   when /^.:scan=>\["([^"]+)"/ then
     puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
   when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
     reduce_line = $2.to_i
   when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
@@ -172,7 +200,7 @@ ARGF.each_line do |line|
   when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
     stack << "none" if stack.empty?
     item = munge $1
-    x = stack.map { |s| s.strip }.join " "
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
     if x != item then # prevent kdef -> kdef
       if $v && reduce_line then
         puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "

data/tools/ripper.rb

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby -ws
 
-$d ||= false
-$p ||= false
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
 
 require "ripper/sexp"
 require "pp" if $p
@@ -19,15 +20,19 @@ end
 
 ARGV.each do |path|
   src = path == "-" ? $stdin.read : File.read(path)
-  rip = MySexpBuilder.new src
-  rip.yydebug = $d
 
-  sexp = rip.parse
-
-  if rip.error? then
-    warn "skipping"
-    next
-  end
+  sexp = if $b then
+           Ripper.sexp src
+         else
+           rip = MySexpBuilder.new src
+           rip.yydebug = $d
+           rip.parse
+
+           if rip.error? then
+             warn "skipping"
+             next
+           end
+         end
 
   puts "accept"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.17.0
+  version: 3.18.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
   d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
   KToW560QIey7SPfHWduzFJnV
   -----END CERTIFICATE-----
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -37,20 +37,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.15'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.15.1
+        version: '4.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.15'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.15.1
+        version: '4.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -77,14 +71,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: racc
   requirement: !ruby/object:Gem::Requirement
@@ -160,6 +154,7 @@ description: |-
   * 1.8 parser is at 99.9739% accuracy, 3.651 sigma
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
+  * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -171,6 +166,7 @@ extra_rdoc_files:
 - Manifest.txt
 - README.rdoc
 - debugging.md
+- gauntlet.md
 files:
 - ".autotest"
 - History.rdoc
@@ -181,6 +177,7 @@ files:
 - bin/ruby_parse_extract_error
 - compare/normalize.rb
 - debugging.md
+- gauntlet.md
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
@@ -202,9 +199,11 @@ files:
 - lib/ruby27_parser.y
 - lib/ruby30_parser.rb
 - lib/ruby30_parser.y
+- lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
+- lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
 - lib/ruby_parser.yy
 - lib/ruby_parser_extras.rb