ripper_ruby_parser 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc516bb8bd2fd0f9fd310f848dbf272481d6c15a8ee3aeecba41ac8e24a7b8c3
-  data.tar.gz: 2d79ba2d094e3694993f48e92565c7424121331aad653bd6cb5fad51a7f9ea75
+  metadata.gz: 20d34b3ba8f1c186e8df1118334a231d2520c9ef51a36b4c7d5b12c33fe9947b
+  data.tar.gz: 881e3286d632c0189a33a2add98dd43bddda798cf6c7c8f3fa75e1b9dfdda7e3
 SHA512:
-  metadata.gz: 2d521a088a674f96b3639d624de7ababb6dbea93965fee1185b6a14f8a289ef88566a87cc5918004d3765f19de7ea0a15d99cfb30792ff519e1d8426555bb162
-  data.tar.gz: 0a79e0832ce26282bacb13d534dde0460f33b677e7a07c23bde4d530848228bda65460ce2c229a97ddc25c5f12cc0aa75ea176f09e3959fcbf78e34536fa6bcd
+  metadata.gz: f3a58dd16a10a8c0f64996c753e2da3c02d7b0ec074099795bf240925e388a5edb215edf680df0c70baab82c42ea77d22762d2a2b27a7a9a246aa01bb47753a1
+  data.tar.gz: 7b360aaea5ba1fe76855a4e534d56aec13b1a470f7cc9e539d4e2c996ef0684848142ea6c4d2e522c439e84d62628ebaf571da5c993e9a44a0f9641d8e3283df

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.6.1 / 2019-04-22
+
+* Improve line numbering for some block structures ([#82])
+  - Fix line numbering for empty method bodies
+  - Assign correct line numbers to END blocks
+  - Assign correct line numbers to begin blocks
+  - Assign correct line numbers to BEGIN blocks
+* Fix line numbering for several literals ([#80])
+  - Fix line numbering for plain regexp literals
+  - Fix line numbering for backtick literals
+  - Fix line numbering for simple string literals
+  - Fix line numbering for keyword-like symbols
+
 ## 1.6.0 / 2019-04-12
 
 * Fix line numbering for range literals ([#79])
@@ -158,6 +171,8 @@
 * Initial release
 
 <!-- Pull request links -->
+[#82]: https://github.com/mvz/ripper_ruby_parser/pull/82
+[#80]: https://github.com/mvz/ripper_ruby_parser/pull/80
 [#79]: https://github.com/mvz/ripper_ruby_parser/pull/79
 [#78]: https://github.com/mvz/ripper_ruby_parser/pull/78
 [#77]: https://github.com/mvz/ripper_ruby_parser/pull/77

data/README.md

@@ -22,14 +22,14 @@ The following incompatibilities cannot be changed:
 
 * RipperRubyParser won't handle non-UTF-8 files without an encoding comment,
   just like regular Ruby
-* RipperRubyParser does not attempt to match RubyParser's line numbering bugs
+* RipperRubyParser does not match RubyParser's line numbering
 * RipperRubyParser correctly dedents heredocs with interpolations
 
 The following incompatibilities can be made compatible by turning on
 extra-compatible mode:
 
 * Operator assignment of a method call without parentheses to a collection
-  element uses and `:array` S-expression instead of `:arglist`
+  element uses an `:array` S-expression instead of `:arglist`
 * RipperRubyParser keeps carriage return characters in heredocs that include
   them
 

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -32,6 +32,15 @@ module RipperRubyParser
       super
     end
 
+    def on_begin(*args)
+      commentize(:begin, super)
+    end
+
+    def on_void_stmt
+      result = super
+      result << [lineno, column]
+    end
+
     def on_comment(tok)
       @comment += tok
     end
@@ -49,14 +58,15 @@ module RipperRubyParser
     end
 
     def on_kw(tok)
+      result = super
       case tok
-      when 'class', 'def', 'module'
+      when 'class', 'def', 'module', 'BEGIN', 'begin', 'END'
         unless @in_symbol
-          @comment_stack.push [tok.to_sym, @comment]
+          @comment_stack.push [result, @comment]
           @comment = ''
         end
       end
-      super
+      result
     end
 
     def on_module(*args)
@@ -275,6 +285,14 @@ module RipperRubyParser
       super
     end
 
+    def on_BEGIN(*args)
+      commentize(:BEGIN, super)
+    end
+
+    def on_END(*args)
+      commentize(:END, super)
+    end
+
     def on_parse_error(*args)
       raise SyntaxError, *args
     end
@@ -298,8 +316,9 @@ module RipperRubyParser
     private
 
     def commentize(_name, exp)
-      _tok, comment = @comment_stack.pop
+      (_, _kw, loc), comment = @comment_stack.pop
       @comment = ''
+      exp.push loc
       [:comment, comment, exp]
     end
   end

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -47,10 +47,10 @@ module RipperRubyParser
       end
 
       def process_begin(exp)
-        _, body = exp.shift 2
+        _, body, pos = exp.shift 3
 
         body = convert_empty_to_nil_symbol process(body)
-        s(:begin, body)
+        with_position pos, s(:begin, body)
       end
 
       def process_rescue(exp)
@@ -86,7 +86,9 @@ module RipperRubyParser
 
         body = s()
 
-        main = wrap_in_block map_process_list_compact main.sexp_body
+        main_list = map_unwrap_begin_list map_process_list main.sexp_body
+        line = main_list.first.line
+        main = wrap_in_block reject_void_stmt main_list
         body << main if main
 
         if rescue_block
@@ -102,7 +104,7 @@ module RipperRubyParser
           body = s(s(:ensure, *body))
         end
 
-        wrap_in_block(body) || s()
+        wrap_in_block(body) || s().line(line)
       end
 
       def process_rescue_mod(exp)

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -11,10 +11,10 @@ module RipperRubyParser
 
       def process_string_content(exp)
         _, *rest = shift_all exp
-        string, rest = extract_string_parts(rest)
+        line, string, rest = extract_string_parts(rest)
 
         if rest.empty?
-          s(:str, string)
+          with_line_number(line, s(:str, string))
         else
           s(:dstr, string, *rest)
         end
@@ -63,9 +63,9 @@ module RipperRubyParser
 
       def process_xstring(exp)
         _, *rest = shift_all exp
-        string, rest = extract_string_parts(rest)
+        line, string, rest = extract_string_parts(rest)
         if rest.empty?
-          s(:xstr, string)
+          s(:xstr, string).line(line)
         else
           s(:dxstr, string, *rest)
         end
@@ -77,7 +77,9 @@ module RipperRubyParser
         content = process(content)
         numflags = character_flags_to_numerical flags
 
-        return s(:lit, Regexp.new(content.last, numflags)) if content.length == 2
+        if content.length == 2
+          return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
+        end
 
         content.sexp_type = :dregx_once if flags =~ /o/
         content << numflags unless numflags == 0
@@ -86,8 +88,8 @@ module RipperRubyParser
 
       def process_regexp(exp)
         _, *rest = shift_all exp
-        string, rest = extract_string_parts(rest)
-        s(:dregx, string, *rest)
+        line, string, rest = extract_string_parts(rest)
+        with_line_number(line, s(:dregx, string, *rest))
       end
 
       def process_symbol_literal(exp)
@@ -126,10 +128,10 @@ module RipperRubyParser
       REGEXP_LITERALS = ['/', /^%r.$/].freeze
 
       def process_at_tstring_content(exp)
-        _, content, _, delim = exp.shift 4
+        _, content, pos, delim = exp.shift 4
         string = handle_string_unescaping(content, delim)
         string = handle_string_encoding(string, delim)
-        s(:str, string)
+        with_position(pos, s(:str, string))
       end
 
       def process_array(exp)
@@ -164,7 +166,7 @@ module RipperRubyParser
       private
 
       def extract_string_parts(list)
-        return '', [] if list.empty?
+        return nil, '', [] if list.empty?
 
         list = merge_raw_string_literals list
         list = map_process_list list
@@ -185,10 +187,11 @@ module RipperRubyParser
         string = ''
         while parts.first&.sexp_type == :str
           str = parts.shift
+          line ||= str.line
           string += str.last
         end
 
-        return string, parts
+        return line, string, parts
       end
 
       def merge_raw_string_literals(list)
@@ -230,12 +233,13 @@ module RipperRubyParser
       def handle_symbol_content(node)
         if node.sexp_type == :'@kw'
           symbol, position = extract_node_symbol_with_position(node)
+          with_position(position, s(:lit, symbol))
         else
           processed = process(node)
           symbol = processed.last.to_sym
-          position = processed.line
+          line = processed.line
+          with_line_number(line, s(:lit, symbol))
         end
-        with_line_number(position, s(:lit, symbol))
       end
 
       def merge_left_into_right(left, right)

data/lib/ripper_ruby_parser/sexp_handlers/loops.rb

@@ -52,10 +52,12 @@ module RipperRubyParser
       def handle_conditional_loop_mod(type, negated_type, exp)
         _, cond, body = exp.shift 3
 
+        cond = process(cond)
+        body = process(body)
         check_at_start = check_at_start?(body)
         construct_conditional_loop(type, negated_type,
-                                   unwrap_begin(process(cond)),
-                                   unwrap_begin(process(body)),
+                                   unwrap_begin(cond),
+                                   unwrap_begin(body),
                                    check_at_start)
       end
 

data/lib/ripper_ruby_parser/sexp_handlers/methods.rb

@@ -5,9 +5,9 @@ module RipperRubyParser
     # Sexp handers for method definitions and related constructs
     module Methods
       def process_def(exp)
-        _, ident, params, body = exp.shift 4
+        _, ident, params, body, pos = exp.shift 5
 
-        ident, pos = extract_node_symbol_with_position ident
+        ident, = extract_node_symbol_with_position ident
 
         in_method do
           params = convert_arguments(process(params))
@@ -19,7 +19,7 @@ module RipperRubyParser
       end
 
       def process_defs(exp)
-        _, receiver, _, ident, params, body = exp.shift 6
+        _, receiver, _, ident, params, body, = exp.shift 7
 
         ident, = extract_node_symbol_with_position ident
 
@@ -85,7 +85,7 @@ module RipperRubyParser
         block = process exp
         case block.length
         when 0
-          [s(:nil)]
+          [s(:nil).line(block.line)]
         else
           unwrap_block block
         end

data/lib/ripper_ruby_parser/sexp_processor.rb

@@ -45,6 +45,8 @@ module RipperRubyParser
       @in_method_body = false
       @kwrest = []
       @block_kwrest = []
+
+      @kept_comment = nil
     end
 
     include SexpHandlers
@@ -56,23 +58,23 @@ module RipperRubyParser
     end
 
     def process_module(exp)
-      _, const_ref, body = exp.shift 3
-      const, line = const_ref_to_const_with_line_number const_ref
-      with_line_number(line,
-                       s(:module, const, *class_or_module_body(body)))
+      _, const_ref, body, pos = exp.shift 4
+      const = const_ref_to_const const_ref
+      with_position(pos,
+                    s(:module, const, *class_or_module_body(body)))
     end
 
     def process_class(exp)
-      _, const_ref, parent, body = exp.shift 4
-      const, line = const_ref_to_const_with_line_number const_ref
+      _, const_ref, parent, body, pos = exp.shift 5
+      const = const_ref_to_const const_ref
       parent = process(parent)
-      with_line_number(line,
-                       s(:class, const, parent, *class_or_module_body(body)))
+      with_position(pos,
+                    s(:class, const, parent, *class_or_module_body(body)))
     end
 
     def process_sclass(exp)
-      _, klass, block = exp.shift 3
-      s(:sclass, process(klass), *class_or_module_body(block))
+      _, klass, block, pos = exp.shift 4
+      with_position pos, s(:sclass, process(klass), *class_or_module_body(block))
     end
 
     def process_stmts(exp)
@@ -104,6 +106,11 @@ module RipperRubyParser
       s(:valias, left[1].to_sym, right[1].to_sym)
     end
 
+    def process_void_stmt(exp)
+      _, pos = exp.shift 2
+      with_position pos, s(:void_stmt)
+    end
+
     def process_const_path_ref(exp)
       _, left, right = exp.shift 3
       s(:colon2, process(left), extract_node_symbol(process(right)))
@@ -139,21 +146,28 @@ module RipperRubyParser
 
     def process_comment(exp)
       _, comment, inner = exp.shift 3
+      comment = @kept_comment + comment if @kept_comment
+      @kept_comment = nil
       sexp = process(inner)
-      sexp.comments = comment
+      case sexp.sexp_type
+      when :defs, :defn, :module, :class, :sclass
+        sexp.comments = comment
+      else
+        @kept_comment = comment
+      end
       sexp
     end
 
     def process_BEGIN(exp)
-      _, body = exp.shift 2
+      _, body, pos = exp.shift 3
       body = reject_void_stmt map_process_list body.sexp_body
-      s(:iter, s(:preexe), 0, *body)
+      with_position pos, s(:iter, s(:preexe), 0, *body)
     end
 
     def process_END(exp)
-      _, body = exp.shift 2
+      _, body, pos = exp.shift 3
       body = map_process_list_compact body.sexp_body
-      s(:iter, s(:postexe), 0, *body)
+      with_position pos, s(:iter, s(:postexe), 0, *body)
     end
 
     # number literals
@@ -242,11 +256,10 @@ module RipperRubyParser
 
     private
 
-    def const_ref_to_const_with_line_number(const_ref)
+    def const_ref_to_const(const_ref)
       const = process(const_ref)
-      line = const.line
       const = const[1] if const.sexp_type == :const
-      return const, line
+      const
     end
 
     def class_or_module_body(exp)

data/lib/ripper_ruby_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RipperRubyParser
-  VERSION = '1.6.0'
+  VERSION = '1.6.1'
 end

data/test/end_to_end/comparison_test.rb

@@ -9,8 +9,8 @@ describe 'Using RipperRubyParser and RubyParser' do
       "puts 'Hello World'"
     end
 
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
+    it 'gives the same result with line numbers' do
+      program.must_be_parsed_as_before with_line_numbers: true
     end
   end
 
@@ -35,8 +35,8 @@ describe 'Using RipperRubyParser and RubyParser' do
       END
     end
 
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
+    it 'gives the same result with line numbers' do
+      program.must_be_parsed_as_before with_line_numbers: true
     end
   end
 
@@ -45,8 +45,8 @@ describe 'Using RipperRubyParser and RubyParser' do
       'def fred() yield(3) if block_given?; end'
     end
 
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
+    it 'gives the same result with line numbers' do
+      program.must_be_parsed_as_before with_line_numbers: true
     end
   end
 
@@ -97,8 +97,8 @@ describe 'Using RipperRubyParser and RubyParser' do
       "/(\#{@types})\\s*(\\w+)\\s*\\(([^)]*)\\)/"
     end
 
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
+    it 'gives the same result with line numbers' do
+      program.must_be_parsed_as_before with_line_numbers: true
     end
   end
 end

data/test/end_to_end/line_numbering_test.rb

@@ -4,30 +4,6 @@ require File.expand_path('../test_helper.rb', File.dirname(__FILE__))
 require 'ruby_parser'
 
 describe 'Using RipperRubyParser and RubyParser' do
-  def to_line_numbers(exp)
-    exp.map! do |sub_exp|
-      if sub_exp.is_a? Sexp
-        to_line_numbers sub_exp
-      else
-        sub_exp
-      end
-    end
-
-    if exp.sexp_type == :scope
-      exp
-    else
-      s(:line_number, exp.line, exp)
-    end
-  end
-
-  let :newparser do
-    RipperRubyParser::Parser.new
-  end
-
-  let :oldparser do
-    RubyParser.for_current_ruby
-  end
-
   describe 'for a multi-line program' do
     let :program do
       <<-END
@@ -44,21 +20,12 @@ describe 'Using RipperRubyParser and RubyParser' do
       END
     end
 
-    let :original do
-      oldparser.parse program
-    end
-
-    let :imitation do
-      newparser.parse program
-    end
-
     it 'gives the same result' do
-      imitation.must_equal original
+      program.must_be_parsed_as_before
     end
 
     it 'gives the same result with line numbers' do
-      formatted(to_line_numbers(imitation)).
-        must_equal formatted(to_line_numbers(original))
+      program.must_be_parsed_as_before with_line_numbers: true
     end
   end
 end

data/test/ripper_ruby_parser/commenting_ripper_parser_test.rb

@@ -22,7 +22,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:def,
                                 s(:@ident, 'foo', s(2, 4)),
                                 empty_params_list,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(2, 12))), nil, nil, nil),
+                                s(2, 0)))))
     end
 
     it 'produces a blank comment node surrounding a def that has no comment' do
@@ -34,7 +36,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:def,
                                 s(:@ident, 'foo', s(1, 4)),
                                 empty_params_list,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(1, 12))), nil, nil, nil),
+                                s(1, 0)))))
     end
 
     it 'produces a comment node surrounding a commented class' do
@@ -46,7 +50,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:class,
                                 s(:const_ref, s(:@const, 'Foo', s(2, 6))),
                                 nil,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(2, 10))), nil, nil, nil),
+                                s(2, 0)))))
     end
 
     it 'produce a blank comment node surrounding a class that has no comment' do
@@ -58,7 +64,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:class,
                                 s(:const_ref, s(:@const, 'Foo', s(1, 6))),
                                 nil,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(1, 10))), nil, nil, nil),
+                                s(1, 0)))))
     end
 
     it 'produces a comment node surrounding a commented module' do
@@ -69,7 +77,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               "# Foo\n",
                               s(:module,
                                 s(:const_ref, s(:@const, 'Foo', s(2, 7))),
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(2, 11))), nil, nil, nil),
+                                s(2, 0)))))
     end
 
     it 'produces a blank comment node surrounding a module that has no comment' do
@@ -80,7 +90,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               '',
                               s(:module,
                                 s(:const_ref, s(:@const, 'Foo', s(1, 7))),
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(1, 11))), nil, nil, nil),
+                                s(1, 0)))))
     end
 
     it 'is not confused by a symbol containing a keyword' do
@@ -93,7 +105,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:def,
                                 s(:@ident, 'foo', s(1, 12)),
                                 empty_params_list,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(1, 20))), nil, nil, nil),
+                                s(1, 8)))))
     end
 
     it 'is not confused by a dynamic symbol' do
@@ -107,7 +121,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:def,
                                 s(:@ident, 'bar', s(1, 12)),
                                 empty_params_list,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(1, 20))), nil, nil, nil),
+                                s(1, 8)))))
     end
 
     it 'is not confused by a dynamic symbol containing a class definition' do
@@ -125,10 +141,8 @@ describe RipperRubyParser::CommentingRipperParser do
                                         s(:const_ref, s(:@const, 'Bar', s(1, 13))),
                                         nil,
                                         s(:bodystmt,
-                                          s(:stmts, s(:void_stmt)),
-                                          nil,
-                                          nil,
-                                          nil)))))))))
+                                          s(:stmts, s(:void_stmt, s(1, 17))), nil, nil, nil),
+                                        s(1, 7)))))))))
     end
 
     it 'turns an embedded document into a comment node' do
@@ -140,7 +154,9 @@ describe RipperRubyParser::CommentingRipperParser do
                               s(:class,
                                 s(:const_ref, s(:@const, 'Foo', s(4, 6))),
                                 nil,
-                                s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil)))))
+                                s(:bodystmt,
+                                  s(:stmts, s(:void_stmt, s(4, 10))), nil, nil, nil),
+                                s(4, 0)))))
     end
   end
 

data/test/ripper_ruby_parser/parser_test.rb

@@ -143,6 +143,22 @@ describe RipperRubyParser::Parser do
         'END { }'.
           must_be_parsed_as s(:iter, s(:postexe), 0)
       end
+
+      it 'assigns correct line numbers' do
+        "END {\nfoo\n}".
+          must_be_parsed_as s(:iter,
+                              s(:postexe).line(1), 0,
+                              s(:call, nil, :foo).line(2)).line(1),
+                            with_line_numbers: true
+      end
+
+      it 'assigns correct line numbers to a embedded begin block' do
+        "END {\nbegin\nfoo\nend\n}".
+          must_be_parsed_as s(:iter,
+                              s(:postexe).line(1), 0,
+                              s(:call, nil, :foo).line(3)).line(1),
+                            with_line_numbers: true
+      end
     end
 
     describe 'for the BEGIN keyword' do
@@ -155,6 +171,23 @@ describe RipperRubyParser::Parser do
         'BEGIN { }'.
           must_be_parsed_as s(:iter, s(:preexe), 0)
       end
+
+      it 'assigns correct line numbers' do
+        "BEGIN {\nfoo\n}".
+          must_be_parsed_as s(:iter,
+                              s(:preexe).line(1), 0,
+                              s(:call, nil, :foo).line(2)).line(1),
+                            with_line_numbers: true
+      end
+
+      it 'assigns correct line numbers to a embedded begin block' do
+        "BEGIN {\nbegin\nfoo\nend\n}".
+          must_be_parsed_as s(:iter,
+                              s(:preexe).line(1), 0,
+                              s(:begin,
+                                s(:call, nil, :foo).line(3)).line(2)).line(1),
+                            with_line_numbers: true
+      end
     end
 
     describe 'for constant lookups' do
@@ -300,6 +333,35 @@ describe RipperRubyParser::Parser do
                               s(:call, nil, :baz)))
         result.comments.must_equal "# Foo\n"
       end
+
+      # TODO: Prefer assigning comment to the BEGIN instead
+      it 'assigns comments on BEGIN blocks to the following item' do
+        result = parser.parse "# Bar\nBEGIN { }\n# Foo\nclass Bar\n# foo\ndef foo; end\nend"
+        result.must_equal s(:block,
+                            s(:iter, s(:preexe), 0),
+                            s(:class, :Bar, nil,
+                              s(:defn, :foo, s(:args), s(:nil))))
+        result[2].comments.must_equal "# Bar\n# Foo\n"
+        result[2][3].comments.must_equal "# foo\n"
+      end
+
+      it 'assigns comments on multiple BEGIN blocks to the following item' do
+        result = parser.parse "# Bar\nBEGIN { }\n# Baz\nBEGIN { }\n# Foo\ndef foo; end"
+        result.must_equal s(:block,
+                            s(:iter, s(:preexe), 0),
+                            s(:iter, s(:preexe), 0),
+                            s(:defn, :foo, s(:args), s(:nil)))
+        result[3].comments.must_equal "# Bar\n# Baz\n# Foo\n"
+      end
+
+      it 'assigns comments on BEGIN blocks to the first following item' do
+        result = parser.parse "# Bar\nBEGIN { }\n# Baz\nBEGIN { }\n# Foo\ndef foo; end"
+        result.must_equal s(:block,
+                            s(:iter, s(:preexe), 0),
+                            s(:iter, s(:preexe), 0),
+                            s(:defn, :foo, s(:args), s(:nil)))
+        result[3].comments.must_equal "# Bar\n# Baz\n# Foo\n"
+      end
     end
 
     # Note: differences in the handling of line numbers are not caught by
@@ -359,11 +421,11 @@ describe RipperRubyParser::Parser do
       end
 
       it 'works for a local variable' do
-        result = parser.parse "foo = bar\nfoo\n"
-        result.sexp_type.must_equal :block
-        result[1].line.must_equal 1
-        result[2].line.must_equal 2
-        result.line.must_equal 1
+        "foo = bar\nfoo\n".
+          must_be_parsed_as s(:block,
+                              s(:lasgn, :foo, s(:call, nil, :bar).line(1)).line(1),
+                              s(:lvar, :foo).line(2)).line(1),
+                            with_line_numbers: true
       end
 
       it 'works for an integer literal' do
@@ -386,6 +448,31 @@ describe RipperRubyParser::Parser do
         result.line.must_equal 1
       end
 
+      it 'works for a symbol literal' do
+        result = parser.parse ':foo'
+        result.line.must_equal 1
+      end
+
+      it 'works for a keyword-like symbol literal' do
+        result = parser.parse ':and'
+        result.line.must_equal 1
+      end
+
+      it 'works for a string literal' do
+        result = parser.parse '"foo"'
+        result.line.must_equal 1
+      end
+
+      it 'works for a backtick string literal' do
+        result = parser.parse '`foo`'
+        result.line.must_equal 1
+      end
+
+      it 'works for a plain regexp literal' do
+        result = parser.parse '/foo/'
+        result.line.must_equal 1
+      end
+
       it 'works for a regular expression back reference' do
         result = parser.parse '$1'
         result.line.must_equal 1
@@ -422,15 +509,12 @@ describe RipperRubyParser::Parser do
       end
 
       it 'assigns line numbers to nested sexps without their own line numbers' do
-        result = parser.parse "foo(bar) do\nnext baz\nend\n"
-        result.must_equal s(:iter,
-                            s(:call, nil, :foo, s(:call, nil, :bar)),
-                            0,
-                            s(:next, s(:call, nil, :baz)))
-        arglist = result[1][3]
-        block = result[3]
-        nums = [arglist.line, block.line]
-        nums.must_equal [1, 2]
+        "foo(bar) do\nnext baz\nend\n".
+          must_be_parsed_as s(:iter,
+                              s(:call, nil, :foo, s(:call, nil, :bar).line(1)).line(1),
+                              0,
+                              s(:next, s(:call, nil, :baz).line(2)).line(2)).line(1),
+                            with_line_numbers: true
       end
 
       describe 'when a line number is passed' do

data/test/ripper_ruby_parser/sexp_handlers/methods_test.rb

@@ -204,6 +204,15 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:defn, :for, s(:args),
                               s(:nil))
       end
+
+      it 'assigns correct line numbers when the body is empty' do
+        "def bar\nend".
+          must_be_parsed_as s(:defn,
+                              :bar,
+                              s(:args).line(1),
+                              s(:nil).line(2)).line(1),
+                            with_line_numbers: true
+      end
     end
 
     describe 'for singleton method definitions' do

data/test/ripper_ruby_parser/sexp_processor_test.rb

@@ -74,10 +74,10 @@ describe RipperRubyParser::SexpProcessor do
     end
 
     describe 'for a :module sexp' do
-      it 'does not create body eleents for an empty definition' do
+      it 'does not create body elements for an empty definition' do
         sexp = s(:module,
                  s(:const_ref, s(:@const, 'Foo', s(1, 13))),
-                 s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil))
+                 s(:bodystmt, s(:stmts, s(:void_stmt, s(2, 3))), nil, nil, nil))
         result = processor.process sexp
         result.must_equal s(:module, :Foo)
       end
@@ -103,7 +103,7 @@ describe RipperRubyParser::SexpProcessor do
       it 'does not create body eleents for an empty definition' do
         sexp = s(:class,
                  s(:const_ref, s(:@const, 'Foo', s(1, 13))), nil,
-                 s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil))
+                 s(:bodystmt, s(:stmts, s(:void_stmt, s(2, 8))), nil, nil, nil))
         result = processor.process sexp
         result.must_equal s(:class, :Foo, nil)
       end
@@ -128,7 +128,7 @@ describe RipperRubyParser::SexpProcessor do
         sexp = s(:class,
                  s(:const_ref, s(:@const, 'Foo', s(1, 13))),
                  s(:var_ref, s(:@const, 'Bar', s(1, 12))),
-                 s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil))
+                 s(:bodystmt, s(:stmts, s(:void_stmt, s(2, 7))), nil, nil, nil))
         result = processor.process sexp
         result.must_equal s(:class, :Foo, s(:const, :Bar))
       end
@@ -153,7 +153,7 @@ describe RipperRubyParser::SexpProcessor do
         sexp = s(:def,
                  s(:@ident, 'foo', s(1, 4)),
                  s(:params, nil, nil, nil, nil, nil),
-                 s(:bodystmt, s(:stmts, s(:void_stmt)), nil, nil, nil))
+                 s(:bodystmt, s(:stmts, s(:void_stmt, s(2, 3))), nil, nil, nil))
         result = processor.process sexp
         result.must_equal s(:defn, :foo, s(:args), s(:nil))
       end

data/test/test_helper.rb

@@ -13,14 +13,36 @@ require 'ripper_ruby_parser'
 
 module MiniTest
   class Spec
-    def formatted(exp)
-      exp.to_s.gsub(/\), /, "),\n")
+    def inspect_with_line_numbers(exp)
+      parts = exp.map do |sub_exp|
+        if sub_exp.is_a? Sexp
+          inspect_with_line_numbers(sub_exp)
+        else
+          sub_exp.inspect
+        end
+      end
+
+      plain = "s(#{parts.join(', ')})"
+      if exp.line
+        "#{plain}.line(#{exp.line})"
+      else
+        plain
+      end
+    end
+
+    def formatted(exp, with_line_numbers: false)
+      inspection = if with_line_numbers
+                     inspect_with_line_numbers(exp)
+                   else
+                     exp.inspect
+                   end
+      inspection.gsub(/\), /, "),\n")
     end
 
     def fix_lines(exp)
-      return s(:lit, :__LINE__) if exp.sexp_type == :lit && exp.line == exp[1]
+      return s(:lit, :__LINE__).line(exp.line) if exp.sexp_type == :lit && exp.line == exp[1]
 
-      inner = exp.map do |sub_exp|
+      exp.sexp_body = exp.sexp_body.map do |sub_exp|
         if sub_exp.is_a? Sexp
           fix_lines sub_exp
         else
@@ -28,11 +50,13 @@ module MiniTest
         end
       end
 
-      s(*inner)
+      exp
     end
 
     def to_comments(exp)
-      inner = exp.map do |sub_exp|
+      comments = exp.comments.to_s.gsub(/\n\s*\n/, "\n")
+
+      exp.sexp_body = exp.sexp_body.map do |sub_exp|
         if sub_exp.is_a? Sexp
           to_comments sub_exp
         else
@@ -40,15 +64,14 @@ module MiniTest
         end
       end
 
-      comments = exp.comments.to_s.gsub(/\n\s*\n/, "\n")
       if comments.empty?
-        s(*inner)
+        exp
       else
-        s(:comment, comments, s(*inner))
+        s(:comment, comments, exp)
       end
     end
 
-    def assert_parsed_as(sexp, code, extra_compatible: false)
+    def assert_parsed_as(sexp, code, extra_compatible: false, with_line_numbers: false)
       parser = RipperRubyParser::Parser.new
       parser.extra_compatible = extra_compatible
       result = parser.parse code
@@ -56,11 +79,12 @@ module MiniTest
         assert_nil result
       else
         assert_equal sexp, result
-        assert_equal sexp.to_s, result.to_s
+        assert_equal(formatted(sexp, with_line_numbers: with_line_numbers),
+                     formatted(result, with_line_numbers: with_line_numbers))
       end
     end
 
-    def assert_parsed_as_before(code)
+    def assert_parsed_as_before(code, with_line_numbers: false)
       oldparser = RubyParser.for_current_ruby
       newparser = RipperRubyParser::Parser.new
       newparser.extra_compatible = true
@@ -68,7 +92,9 @@ module MiniTest
       result = newparser.parse code
       expected = to_comments fix_lines expected
       result = to_comments fix_lines result
-      assert_equal formatted(expected), formatted(result)
+      assert_equal expected, result
+      assert_equal(formatted(expected, with_line_numbers: with_line_numbers),
+                   formatted(result, with_line_numbers: with_line_numbers))
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ripper_ruby_parser
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Matijs van Zuijlen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-12 00:00:00.000000000 Z
+date: 2019-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor