ruby_parser 3.20.2 → 3.21.0

data/lib/ruby_parser_extras.rb

@@ -30,11 +30,11 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.20.2"
+  VERSION = "3.21.0"
 
-  attr_accessor :lexer, :in_def, :in_single, :file
+  attr_accessor :lexer, :in_def, :in_single, :file, :in_argdef
   attr_accessor :in_kwarg
-  attr_reader :env, :comments
+  attr_reader :env
 
   ##
   # Canonicalize conditionals. Eg:
@@ -92,6 +92,7 @@ module RubyParserStuff
     [k, true]
   }.to_h
 
+  # TODO: remove
   has_enc = "".respond_to? :encoding
 
   # This is in sorted order of occurrence according to
@@ -122,9 +123,9 @@ module RubyParserStuff
     self.lexer = RubyLexer.new v && v.to_i
     self.lexer.parser = self
     self.in_kwarg = false
+    self.in_argdef = false
 
     @env = RubyParserStuff::Environment.new
-    @comments = []
 
     @canonicalize_conditions = true
 
@@ -566,7 +567,7 @@ module RubyParserStuff
 
   def handle_encoding str
     str = str.dup
-    has_enc = str.respond_to? :encoding
+    has_enc = str.respond_to? :encoding # TODO: remove
     encoding = nil
 
     header = str.each_line.first(2)
@@ -928,7 +929,7 @@ module RubyParserStuff
   end
 
   def new_class val
-    (_, line), path, superclass, _, body, (_, line_max) = val
+    (_, line, comment), path, superclass, _, body, (_, line_max) = val
 
     path = path.first if path.instance_of? Array
 
@@ -944,7 +945,7 @@ module RubyParserStuff
 
     result.line = line
     result.line_max = line_max
-    result.comments = self.comments.pop
+    result.comments = comment if comment
     result
   end
 
@@ -972,7 +973,11 @@ module RubyParserStuff
   end
 
   def new_defn val
-    _, (name, line), in_def, args, body, (_, line_max) = val
+    if val.size == 4 then
+      ((_, line, comment), (name, _line, in_def)), args, body, (_, line_max) = val
+    else
+      (_, line, comment), (name, line), in_def, args, body, (_, line_max) = val
+    end
 
     body ||= s(:nil).line line
 
@@ -987,13 +992,14 @@ module RubyParserStuff
       result.push body
     end
 
-    result.comments = self.comments.pop
+    result.comments = comment if comment
 
     [result, in_def]
   end
 
   def new_endless_defn val
-    (name, line, in_def), args, _, body, _, resbody = val
+    # not available in 2.x so we don't need to check size
+    ((_, line, comment), (name, _, in_def)), args, _, body, _, resbody = val
 
     result =
       if resbody then
@@ -1008,13 +1014,15 @@ module RubyParserStuff
     local_pop in_def
     endless_method_name result
 
-    result.comments = self.comments.pop
+    result.comments = comment if comment
 
     result
   end
 
   def new_endless_defs val
-    (recv, (name, line, in_def)), args, _, body, _, resbody = val
+    # not available in 2.x so we don't need to check size
+    ((_, line, comment), recv, _, _, (name, line, in_def)), \
+      args, _, body, _, resbody = val
 
     result =
       if resbody then
@@ -1030,13 +1038,19 @@ module RubyParserStuff
     local_pop in_def
     endless_method_name result
 
-    result.comments = self.comments.pop
+    result.comments = comment if comment
 
     result
   end
 
   def new_defs val
-    _, recv, (name, line), in_def, args, body, (_, line_max) = val
+    if val.size == 4 then
+      ((_, line, comment), recv, _, _, (name, line, in_def)), \
+        args, body, (_, line_max) = val
+    else
+      (_, line, comment), recv, (name, _), in_def, \
+        args, body, (_, line_max) = val
+    end
 
     body ||= s(:nil).line line
 
@@ -1054,7 +1068,7 @@ module RubyParserStuff
       result.push body
     end
 
-    result.comments = self.comments.pop
+    result.comments = comment if comment
 
     [result, in_def]
   end
@@ -1208,7 +1222,7 @@ module RubyParserStuff
   end
 
   def new_module val
-    (_, line_min), path, _, body, (_, line_max) = val
+    (_, line_min, comment), path, _, body, (_, line_max) = val
 
     path = path.first if path.instance_of? Array
 
@@ -1223,7 +1237,7 @@ module RubyParserStuff
       end
     end
 
-    result.comments = self.comments.pop
+    result.comments = comment if comment
     result
   end
 
@@ -1517,11 +1531,16 @@ module RubyParserStuff
     end
   end
 
+  KEEP_COMMENT_TOKENS = [:kCLASS, :kMODULE, :kDEF, :tNL]
+
   def next_token
     token = self.lexer.next_token
 
     if token and token.first != RubyLexer::EOF then
       self.last_token_type = token
+
+      self.lexer.comment = nil unless KEEP_COMMENT_TOKENS.include? token.first
+
       return token
     elsif !token
       return self.lexer.next_token
@@ -1581,7 +1600,6 @@ module RubyParserStuff
     self.in_def = false
     self.in_single = 0
     self.env.reset
-    self.comments.clear
     self.last_token_type = nil
   end
 

data/test/test_ruby_lexer.rb

@@ -584,7 +584,7 @@ class TestRubyLexer < Minitest::Test
                 :tNL,      nil, EXPR_BEG,
                 :tINTEGER, 2,   EXPR_NUM)
 
-    assert_equal "# one\n# two\n", @lex.comments
+    assert_equal "# one\n# two\n", @lex.comment
   end
 
   def test_yylex_comment_begin
@@ -592,13 +592,13 @@ class TestRubyLexer < Minitest::Test
                 nil,
                 :tINTEGER, 42, EXPR_NUM)
 
-    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
+    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comment
   end
 
   def test_yylex_comment_begin_bad
     refute_lex("=begin\nblah\nblah\n")
 
-    assert_equal "", @lex.comments
+    assert_nil @lex.comment
   end
 
   def test_yylex_comment_begin_not_comment
@@ -617,13 +617,13 @@ class TestRubyLexer < Minitest::Test
   def test_yylex_comment_begin_space
     assert_lex3("=begin blah\nblah\n=end\n", nil)
 
-    assert_equal "=begin blah\nblah\n=end\n", @lex.comments
+    assert_equal "=begin blah\nblah\n=end\n", @lex.comment
   end
 
   def test_yylex_comment_end_space_and_text
     assert_lex3("=begin blah\nblah\n=end blab\n", nil)
 
-    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comment
   end
 
   def test_yylex_comment_eos

data/test/test_ruby_parser.rb

@@ -500,7 +500,7 @@ module TestRubyParserShared
     exp = rb.strip + "\n"
 
     refute_parse rb
-    assert_equal exp, processor.lexer.comments
+    assert_equal exp, processor.lexer.comment
   end
 
   def test_bug_cond_pct
@@ -1722,11 +1722,10 @@ module TestRubyParserShared
 
     assert_equal "# class comment\n", sexp.comments
     act = sexp.find_nodes(:defn).map(&:comments)
-    exp = ["", "# Returns a list of things\n"]
+    exp = [nil, "# Returns a list of things\n"]
 
     assert_equal exp, act
-    assert_equal [], processor.comments
-    assert_equal "", processor.lexer.comments
+    assert_nil processor.lexer.comment
   end
 
   def test_parse_if_not_canonical
@@ -5556,6 +5555,14 @@ module TestRubyParserShared31Plus
     assert_case_in rb, pt
   end
 
+  def test_defn_forward_args__no_parens
+    rb = "def f ...\n  m(...)\nend"
+    pt = s(:defn, :f, s(:args, s(:forward_args)),
+          s(:call, nil, :m, s(:forward_args).line(2)).line(2))
+
+    assert_parse rb, pt
+  end
+
   def test_case_in_carat_nonlocal_vars
     processor.env[:a] = :lvar
 
@@ -5664,6 +5671,10 @@ module TestRubyParserShared32Plus
   end
 end
 
+module TestRubyParserShared33Plus
+  include TestRubyParserShared32Plus
+end
+
 class Minitest::Test
   def skip s = "blah"
     warn "ignoring skip for %s: %s" % [name, s]
@@ -5685,28 +5696,17 @@ class TestRubyParser < Minitest::Test
     pt = s(:call, s(:call, nil, :a), :call)
 
     assert_equal pt, processor.parse(rb)
-
-    # bad syntax
-    e = assert_raises Racc::ParseError do
-      capture_io do
-        processor.parse "a.("
-      end
-    end
-
-    assert_includes e.message, 'parse error on value "$" ($end)'
   end
 
-  def test_parse_error_from_first
+  def test_parse_error
     processor = RubyParser.new
 
+    # bad syntax
     e = assert_raises Racc::ParseError do
-      capture_io do
-        processor.parse "a -> () {"
-      end
+      processor.parse "a.("
     end
 
-    # This is a 2.x error, will fail on 1.8/1.9.
-    assert_includes e.message, 'parse error on value "$" ($end)'
+    assert_match(/parse error on value \S+ ..end./, e.message)
   end
 end
 
@@ -6023,6 +6023,16 @@ class TestRubyParserV32 < RubyParserTestCase
   end
 end
 
+class TestRubyParserV33 < RubyParserTestCase
+  include TestRubyParserShared33Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V33.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do

data/tools/munge.rb

@@ -174,6 +174,10 @@ ARGF.each_line do |line|
     last_token = token
   when /^Reading a token: / then
     next # skip
+  when /^Reading a token$/ then # wtf?
+    next # skip
+  when /^(?:add_delayed_token|parser_dispatch)/ then # dunno what this is yet
+    next # skip
   when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
     token = munge $1
     next if last_token == token
@@ -212,7 +216,9 @@ ARGF.each_line do |line|
     reduce_line = nil
     stack.clear
   when /^reduce/ then # ruby_parser side
-    puts munge line.chomp
+    s = munge line.chomp
+    next if s =~ /reduce\s+(\w+) --> \1/
+    puts s
     puts
   when /^(\w+_stack)\.(\w+)/ then
     # TODO: make pretty, but still informative w/ line numbers etc
@@ -223,7 +229,7 @@ ARGF.each_line do |line|
     # puts line
     # TODO: make pretty, but still informative w/ line numbers etc
     puts line.gsub("true", "1").gsub("false", "0")
-  when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
+  when /^lex_state: :?([\w|()]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
     a, b, c = $1.upcase, $2.upcase, $3
     a.gsub!(/EXPR_/, "")
     b.gsub!(/EXPR_/, "")

data/tools/ripper.rb

@@ -21,18 +21,20 @@ end
 ARGV.each do |path|
   src = path == "-" ? $stdin.read : File.read(path)
 
-  sexp = if $b then
-           Ripper.sexp src
-         else
-           rip = MySexpBuilder.new src
-           rip.yydebug = $d
-           rip.parse
-
-           if rip.error? then
-             warn "skipping"
-             next
-           end
-         end
+  sexp = nil
+
+  if $b then
+    sexp = Ripper.sexp src
+  else
+    rip = MySexpBuilder.new src
+    rip.yydebug = $d
+    sexp = rip.parse
+
+    if rip.error? then
+      warn "skipping"
+      next
+    end
+  end
 
   puts "accept"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.20.2
+  version: 3.21.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBzANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBCDANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTIzMDEwMTA3NTExN1oXDTI0MDEwMTA3NTExN1owRTETMBEGA1UE
+  GRYDY29tMB4XDTI0MDEwMjIxMjEyM1oXDTI1MDEwMTIxMjEyM1owRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQAkg3y+PBnBAPWdxxITm5sPHqdWQgSyCpRA20o4LTuWr8BWhSXBkfQNa7cY6fOn
-  xyM34VPzBFbExv6XOGDfOMFBVaYTHuN9peC/5/umL7kLl+nflXzL2QA7K6LYj5Bg
-  sM574Onr0dZDM6Vn69bzQ7rBIFDfK/OhlPzqKZad4nsdcsVH8ODCiT+ATMIZyz5K
-  WCnNtqlyiWXI8tdTpahDgcUwfcN/oN7v4K8iU5IbLJX6HQ5DKgmKjfb6XyMth16k
-  ROfWo9Uyp8ba/j9eVG14KkYRaLydAY1MNQk2yd3R5CGfeOpD1kttxjoypoUJ2dOG
-  nsNBRuQJ1UfiCG97a6DNm+Fr
+  AQCygvpmncmkiSs9r/Kceo4bBPDszhTv6iBi4LwMReqnFrpNLMOWJw7xi8x+3eL2
+  XS09ZPNOt2zm70KmFouBMgOysnDY4k2dE8uF6B8JbZOO8QfalW+CoNBliefOTcn2
+  bg5IOP7UoGM5lC174/cbDJrJnRG9bzig5FAP0mvsgA8zgTRXQzIUAZEo92D5K7p4
+  B4/O998ho6BSOgYBI9Yk1ttdCtti6Y+8N9+fZESsjtWMykA+WXWeGUScHqiU+gH8
+  S7043fq9EbQdBr2AXdj92+CDwuTfHI6/Hj5FVBDULufrJaan4xUgL70Hvc6pTTeW
+  deKfBjgVAq7EYHu1AczzlUly
   -----END CERTIFICATE-----
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2024-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -45,6 +45,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.16'
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -79,20 +93,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
-- !ruby/object:Gem::Dependency
-  name: racc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -119,14 +119,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). It outputs
@@ -184,35 +184,25 @@ files:
 - lib/.document
 - lib/rp_extensions.rb
 - lib/rp_stringscanner.rb
-- lib/ruby20_parser.rb
-- lib/ruby20_parser.y
-- lib/ruby21_parser.rb
-- lib/ruby21_parser.y
-- lib/ruby22_parser.rb
-- lib/ruby22_parser.y
-- lib/ruby23_parser.rb
-- lib/ruby23_parser.y
-- lib/ruby24_parser.rb
-- lib/ruby24_parser.y
-- lib/ruby25_parser.rb
-- lib/ruby25_parser.y
-- lib/ruby26_parser.rb
-- lib/ruby26_parser.y
-- lib/ruby27_parser.rb
-- lib/ruby27_parser.y
-- lib/ruby30_parser.rb
-- lib/ruby30_parser.y
-- lib/ruby31_parser.rb
-- lib/ruby31_parser.y
-- lib/ruby32_parser.rb
-- lib/ruby32_parser.y
-- lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
 - lib/ruby_lexer.rex.rb
 - lib/ruby_lexer_strings.rb
 - lib/ruby_parser.rb
-- lib/ruby_parser.yy
+- lib/ruby_parser2.yy
+- lib/ruby_parser20.rb
+- lib/ruby_parser21.rb
+- lib/ruby_parser22.rb
+- lib/ruby_parser23.rb
+- lib/ruby_parser24.rb
+- lib/ruby_parser25.rb
+- lib/ruby_parser26.rb
+- lib/ruby_parser27.rb
+- lib/ruby_parser3.yy
+- lib/ruby_parser30.rb
+- lib/ruby_parser31.rb
+- lib/ruby_parser32.rb
+- lib/ruby_parser33.rb
 - lib/ruby_parser_extras.rb
 - test/test_ruby_lexer.rb
 - test/test_ruby_parser.rb
@@ -245,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which