ripper_ruby_parser 1.6.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc516bb8bd2fd0f9fd310f848dbf272481d6c15a8ee3aeecba41ac8e24a7b8c3
-  data.tar.gz: 2d79ba2d094e3694993f48e92565c7424121331aad653bd6cb5fad51a7f9ea75
+  metadata.gz: 799011b06831426c0ebecad641082d58bfaa8da0aea9359a3a2d152ea01bd196
+  data.tar.gz: eec3804031490d2382f59ddc450af14f73b696b9e14f0d2919b64a3df3557853
 SHA512:
-  metadata.gz: 2d521a088a674f96b3639d624de7ababb6dbea93965fee1185b6a14f8a289ef88566a87cc5918004d3765f19de7ea0a15d99cfb30792ff519e1d8426555bb162
-  data.tar.gz: 0a79e0832ce26282bacb13d534dde0460f33b677e7a07c23bde4d530848228bda65460ce2c229a97ddc25c5f12cc0aa75ea176f09e3959fcbf78e34536fa6bcd
+  metadata.gz: eab6dc676f7c5806bcd09f9adf87e5b67dd5320e043c07370014fdfc8e5ca1dffbaa9583754ac74bc354a30c6b27392c3310a7172b20699d475d9122e9dae748
+  data.tar.gz: b776c9d833e7808c7b2ac17ec6bb5a358a93cb1bb925ee94199aeddf85c0cca2e102cdcdfff17ca8d17ac406e0f76c4b981b640085fbd02d2868bca36fdd840d

data/CHANGELOG.md

@@ -1,5 +1,64 @@
 # Changelog
 
+Notable changes to this project will be documented in this file.
+
+This project adheres to [Semantic Versioning 2.0.0][1].
+
+This document is formatted based on [Keep A CHANGELOG][2].
+
+## 1.8.0 / 2021-05-18
+
+### Changes
+
+* Drop support for Ruby 2.4 ([2cf6b4050] and [#145])
+* Update RubyParser compatibility target to 3.15.0 ([#105])
+* Fix handling of dedented heredoc ([#106])
+* Add support for running on Ruby 3.0 ([#123])
+* Various code improvements and other internal changes
+
+## 1.7.2 / 2020-02-28
+
+### Bug fixes
+
+* Support imaginary number literals ([#100])
+* Handle anonymous kwrest arguments ([#95])
+
+### Internal changes
+
+* Update tests ([#101])
+* Prepare for testing on Ruby 2.7 ([#99])
+* Various improvements ([#98])
+* Split long module ([#97])
+
+## 1.7.1 / 2019-11-03
+
+* Handle unicode escapes with five or six hex digits ([#94])
+* Handle safe attribute assignment ([#92])
+* Handle endless ranges on Ruby 2.6+ ([#90])
+* Add preliminary support for Ruby 2.7 ([#89])
+* Improve line number handling for stabby lambdas ([#88])
+
+## 1.7.0 / 2019-11-01
+
+* Make results compatible with RubyParser 3.14.0 ([#85])
+  - Remove obsolete extra-compatible behavior
+  - Parse stabby lambda as :lambda-typed sexp
+  - Emit nil argument for block arguments with trailing comma
+* Require Ruby 2.4 or higher ([#86])
+
+## 1.6.1 / 2019-04-22
+
+* Improve line numbering for some block structures ([#82])
+  - Fix line numbering for empty method bodies
+  - Assign correct line numbers to END blocks
+  - Assign correct line numbers to begin blocks
+  - Assign correct line numbers to BEGIN blocks
+* Fix line numbering for several literals ([#80])
+  - Fix line numbering for plain regexp literals
+  - Fix line numbering for backtick literals
+  - Fix line numbering for simple string literals
+  - Fix line numbering for keyword-like symbols
+
 ## 1.6.0 / 2019-04-12
 
 * Fix line numbering for range literals ([#79])
@@ -158,6 +217,25 @@
 * Initial release
 
 <!-- Pull request links -->
+[#145]: https://github.com/mvz/ripper_ruby_parser/pull/145
+[#123]: https://github.com/mvz/ripper_ruby_parser/pull/123
+[#106]: https://github.com/mvz/ripper_ruby_parser/pull/106
+[#105]: https://github.com/mvz/ripper_ruby_parser/pull/105
+[#101]: https://github.com/mvz/ripper_ruby_parser/pull/101
+[#100]: https://github.com/mvz/ripper_ruby_parser/pull/100
+[#99]: https://github.com/mvz/ripper_ruby_parser/pull/99
+[#98]: https://github.com/mvz/ripper_ruby_parser/pull/98
+[#97]: https://github.com/mvz/ripper_ruby_parser/pull/97
+[#95]: https://github.com/mvz/ripper_ruby_parser/pull/95
+[#94]: https://github.com/mvz/ripper_ruby_parser/pull/94
+[#92]: https://github.com/mvz/ripper_ruby_parser/pull/92
+[#90]: https://github.com/mvz/ripper_ruby_parser/pull/90
+[#89]: https://github.com/mvz/ripper_ruby_parser/pull/89
+[#88]: https://github.com/mvz/ripper_ruby_parser/pull/88
+[#86]: https://github.com/mvz/ripper_ruby_parser/pull/86
+[#85]: https://github.com/mvz/ripper_ruby_parser/pull/85
+[#82]: https://github.com/mvz/ripper_ruby_parser/pull/82
+[#80]: https://github.com/mvz/ripper_ruby_parser/pull/80
 [#79]: https://github.com/mvz/ripper_ruby_parser/pull/79
 [#78]: https://github.com/mvz/ripper_ruby_parser/pull/78
 [#77]: https://github.com/mvz/ripper_ruby_parser/pull/77
@@ -173,3 +251,11 @@
 [#67]: https://github.com/mvz/ripper_ruby_parser/pull/67
 [#66]: https://github.com/mvz/ripper_ruby_parser/pull/66
 [#65]: https://github.com/mvz/ripper_ruby_parser/pull/65
+
+<!-- Git sha links -->
+[2cf6b4050]: https://github.com/mvz/ripper_ruby_parser/commit/2cf6b40501576003b916196e23222276b51aa53c
+
+<!-- Other links -->
+
+[1]:  https://semver.org/spec/v2.0.0.html
+[2]:  https://keepachangelog.com/en/1.0.0/

data/README.md

@@ -10,28 +10,18 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 
 * Drop-in replacement for RubyParser
 * Should handle 1.9 and later syntax gracefully
-* Requires MRI 2.3 or higher
-* Compatible with RubyParser 3.13.1
+* Requires Ruby 2.5 or higher
+* Compatible with RubyParser 3.15.0
 
 ## Known incompatibilities
 
-RipperRubyParser has some incompatibilities with RubyParser. For some of these,
-the behavior can be changed by turning on extra-compatible mode.
-
-The following incompatibilities cannot be changed:
+RipperRubyParser has a few incompatibilities with RubyParser.
 
 * RipperRubyParser won't handle non-UTF-8 files without an encoding comment,
   just like regular Ruby
-* RipperRubyParser does not attempt to match RubyParser's line numbering bugs
-* RipperRubyParser correctly dedents heredocs with interpolations
-
-The following incompatibilities can be made compatible by turning on
-extra-compatible mode:
-
-* Operator assignment of a method call without parentheses to a collection
-  element uses and `:array` S-expression instead of `:arglist`
-* RipperRubyParser keeps carriage return characters in heredocs that include
-  them
+* RipperRubyParser does not always match RubyParser's line numbering
+* RipperRubyParser dedents auto-dedenting heredocs
+* RipperRubyParser does not include postfix comments
 
 ## Install
 
@@ -50,18 +40,11 @@ parser.parse "foo[bar] += baz qux"
 #      s(:arglist, s(:call, nil, :bar)),
 #      :+,
 #      s(:call, nil, :baz, s(:call, nil, :qux)))
-parser.extra_compatible = true
-
-parser.parse "foo[bar] += baz qux"
-# => s(:op_asgn1, s(:call, nil, :foo),
-#      s(:array, s(:call, nil, :bar)),
-#      :+,
-#      s(:call, nil, :baz, s(:call, nil, :qux)))
 ```
 
 ## Requirements
 
-* Ruby 2.3 or higher
+* Ruby 2.5 or higher
 * `sexp_processor`
 
 ## Hacking and contributing
@@ -81,7 +64,7 @@ If you want to send pull requests or patches, please:
 
 (The MIT License)
 
-Copyright (c) 2012, 2014-2019 Matijs van Zuijlen
+Copyright (c) 2012, 2014-2020 Matijs van Zuijlen
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/ripper_ruby_parser.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 
-require 'ripper_ruby_parser/version'
-require 'ripper_ruby_parser/parser'
+require "ripper_ruby_parser/version"
+require "ripper_ruby_parser/parser"

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'ripper'
-require 'ripper_ruby_parser/syntax_error'
-require 'ripper_ruby_parser/unescape'
+require "ripper"
+require "ripper_ruby_parser/syntax_error"
+require "ripper_ruby_parser/unescape"
 
 module RipperRubyParser
   # Variant of Ripper's SexpBuilder parser class that inserts comments as
@@ -12,7 +12,7 @@ module RipperRubyParser
   class CommentingRipperParser < Ripper::SexpBuilder
     def initialize(*args)
       super
-      @comment = ''
+      @comment = ""
       @comment_stack = []
       @delimiter_stack = []
       @space_before = false
@@ -22,16 +22,27 @@ module RipperRubyParser
 
     def parse
       result = super
-      raise 'Ripper parse failed.' unless result
+      raise "Ripper parse failed." unless result
 
       Sexp.from_array(result)
     end
 
+    private
+
     def on_backtick(delimiter)
       @delimiter_stack.push delimiter
       super
     end
 
+    def on_begin(*args)
+      commentize(:begin, super)
+    end
+
+    def on_void_stmt
+      result = super
+      result << [lineno, column]
+    end
+
     def on_comment(tok)
       @comment += tok
     end
@@ -49,14 +60,15 @@ module RipperRubyParser
     end
 
     def on_kw(tok)
+      result = super
       case tok
-      when 'class', 'def', 'module'
+      when "class", "def", "module", "BEGIN", "begin", "END"
         unless @in_symbol
-          @comment_stack.push [tok.to_sym, @comment]
-          @comment = ''
+          @comment_stack.push [result, @comment]
+          @comment = ""
         end
       end
-      super
+      result
     end
 
     def on_module(*args)
@@ -91,6 +103,18 @@ module RipperRubyParser
       @delimiter_stack.push delimiter
     end
 
+    def on_heredoc_dedent(val, width)
+      next_dedent = true
+      val.map! do |e|
+        if e.is_a?(Array) && e[0] == :@tstring_content
+          e = dedent_element(e, width) if next_dedent
+          next_dedent = e[1].end_with? "\n"
+        end
+        e
+      end
+      val
+    end
+
     def on_heredoc_end(_delimiter)
       @delimiter_stack.pop
     end
@@ -245,7 +269,7 @@ module RipperRubyParser
     def on_unary(operator, value)
       if !@space_before && operator == :-@ && NUMBER_LITERAL_TYPES.include?(value.first)
         type, literal, lines = value
-        if literal[0] == '-'
+        if literal[0] == "-"
           super
         else
           [type, "-#{literal}", lines]
@@ -275,31 +299,38 @@ module RipperRubyParser
       super
     end
 
-    def on_parse_error(*args)
-      raise SyntaxError, *args
+    def on_BEGIN(*args)
+      commentize(:BEGIN, super)
     end
 
-    def on_class_name_error(*args)
-      raise SyntaxError, *args
+    def on_END(*args)
+      commentize(:END, super)
     end
 
-    def on_alias_error(*args)
-      raise SyntaxError, *args
+    def on_parse_error(message)
+      raise SyntaxError, message
     end
 
-    def on_assign_error(*args)
-      raise SyntaxError, *args
+    def on_class_name_error(message, *)
+      raise SyntaxError, message
     end
 
-    def on_param_error(*args)
-      raise SyntaxError, *args
+    def on_alias_error(message, *)
+      raise SyntaxError, message
     end
 
-    private
+    def on_assign_error(message, *)
+      raise SyntaxError, message
+    end
+
+    def on_param_error(message, *)
+      raise SyntaxError, message
+    end
 
     def commentize(_name, exp)
-      _tok, comment = @comment_stack.pop
-      @comment = ''
+      (_, _kw, loc), comment = @comment_stack.pop
+      @comment = ""
+      exp.push loc
       [:comment, comment, exp]
     end
   end

data/lib/ripper_ruby_parser/parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'ripper_ruby_parser/commenting_ripper_parser'
-require 'ripper_ruby_parser/sexp_processor'
+require "ripper_ruby_parser/commenting_ripper_parser"
+require "ripper_ruby_parser/sexp_processor"
 
 module RipperRubyParser
   # Main parser class. Brings together Ripper and our
@@ -13,7 +13,7 @@ module RipperRubyParser
       @extra_compatible = false
     end
 
-    def parse(source, filename = '(string)', lineno = 1)
+    def parse(source, filename = "(string)", lineno = 1)
       parser = CommentingRipperParser.new(source, filename, lineno)
       exp = parser.parse
 

data/lib/ripper_ruby_parser/sexp_handlers.rb

@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
-require 'ripper_ruby_parser/sexp_handlers/helper_methods'
+require "ripper_ruby_parser/sexp_handlers/helper_methods"
 
-require 'ripper_ruby_parser/sexp_handlers/assignment'
-require 'ripper_ruby_parser/sexp_handlers/blocks'
-require 'ripper_ruby_parser/sexp_handlers/conditionals'
-require 'ripper_ruby_parser/sexp_handlers/literals'
-require 'ripper_ruby_parser/sexp_handlers/loops'
-require 'ripper_ruby_parser/sexp_handlers/method_calls'
-require 'ripper_ruby_parser/sexp_handlers/methods'
-require 'ripper_ruby_parser/sexp_handlers/operators'
+require "ripper_ruby_parser/sexp_handlers/assignment"
+require "ripper_ruby_parser/sexp_handlers/blocks"
+require "ripper_ruby_parser/sexp_handlers/conditionals"
+require "ripper_ruby_parser/sexp_handlers/literals"
+require "ripper_ruby_parser/sexp_handlers/loops"
+require "ripper_ruby_parser/sexp_handlers/method_calls"
+require "ripper_ruby_parser/sexp_handlers/methods"
+require "ripper_ruby_parser/sexp_handlers/operators"
+require "ripper_ruby_parser/sexp_handlers/string_literals"
 
 module RipperRubyParser
   # Umbrella module for handlers of particular sexp types
@@ -28,6 +29,7 @@ module RipperRubyParser
         include MethodCalls
         include Methods
         include Operators
+        include StringLiterals
       end
     end
   end

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -96,11 +96,10 @@ module RipperRubyParser
         _, lvalue, (_, operator,), value = exp.shift 4
 
         lvalue = process(lvalue)
-        original_value_type = value.sexp_type
         value = process(value)
         operator = operator.chop.to_sym
 
-        create_operator_assignment_sub_type lvalue, value, operator, original_value_type
+        create_operator_assignment_sub_type lvalue, value, operator
       end
 
       private
@@ -123,16 +122,11 @@ module RipperRubyParser
         '&&': :op_asgn_and
       }.freeze
 
-      def create_operator_assignment_sub_type(lvalue, value, operator, original_value_type)
+      def create_operator_assignment_sub_type(lvalue, value, operator)
         case lvalue.sexp_type
         when :aref_field
           _, arr, arglist = lvalue
-          arglist.sexp_type = if extra_compatible &&
-                                 [:command, :command_call].include?(original_value_type)
-                                :array
-                              else
-                                :arglist
-                              end
+          arglist.sexp_type = :arglist
           s(:op_asgn1, arr, arglist, operator, value)
         when :field
           _, obj, _, (_, field) = lvalue
@@ -156,8 +150,13 @@ module RipperRubyParser
           arglist.shift
           s(:attrasgn, arr, :[]=, *arglist)
         when :field
-          _, obj, _, (_, field) = lvalue
-          s(:attrasgn, obj, :"#{field}=", value)
+          _, obj, call_op, (_, field) = lvalue
+          case call_op
+          when :"&.", s(:op, :"&.") # Handle both 2.5 and 2.6 style ops
+            s(:safe_attrasgn, obj, :"#{field}=", value)
+          else
+            s(:attrasgn, obj, :"#{field}=", value)
+          end
         else
           create_assignment_sub_type lvalue, value
         end

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -17,13 +17,14 @@ module RipperRubyParser
       def process_params(exp)
         _, normal, defaults, splat, rest, kwargs, doublesplat, block = exp.shift 8
 
-        args = handle_normal_arguments normal
-        args += handle_default_arguments defaults
-        args += handle_splat splat
-        args += handle_normal_arguments rest
-        args += handle_kwargs kwargs
-        args += handle_double_splat doublesplat
-        args += handle_block_argument block
+        args =
+          handle_normal_arguments(normal) +
+          handle_default_arguments(defaults) +
+          handle_splat(splat) +
+          handle_normal_arguments(rest) +
+          handle_kwargs(kwargs) +
+          handle_double_splat(doublesplat) +
+          handle_block_argument(block)
 
         s(:args, *args)
       end
@@ -35,7 +36,7 @@ module RipperRubyParser
 
       def process_kwrest_param(exp)
         _, sym, = exp.shift 3
-        process(sym)
+        process(sym) || s(:lvar, :"")
       end
 
       def process_block_var(exp)
@@ -47,10 +48,10 @@ module RipperRubyParser
       end
 
       def process_begin(exp)
-        _, body = exp.shift 2
+        _, body, pos = exp.shift 3
 
-        body = convert_empty_to_nil_symbol process(body)
-        s(:begin, body)
+        body = convert_void_stmt_to_nil_symbol process(body)
+        with_position pos, s(:begin, body)
       end
 
       def process_rescue(exp)
@@ -58,51 +59,32 @@ module RipperRubyParser
         rescue_block = map_process_list_compact block.sexp_body
         rescue_block << nil if rescue_block.empty?
 
-        capture = if eclass
-                    if eclass.first.is_a? Symbol
-                      eclass = process(eclass)
-                      body = eclass.sexp_body
-                      if eclass.sexp_type == :mrhs
-                        body.first
-                      else
-                        s(:array, *body)
-                      end
-                    else
-                      s(:array, process(eclass.first))
-                    end
-                  else
-                    s(:array)
-                  end
+        capture = handle_rescue_class_list eclass
 
         capture << create_assignment_sub_type(process(evar), s(:gvar, :$!)) if evar
 
-        s(
-          s(:resbody, capture, *rescue_block),
-          *process(after))
+        s(s(:resbody, capture, *rescue_block), *process(after))
       end
 
       def process_bodystmt(exp)
         _, main, rescue_block, else_block, ensure_block = exp.shift 5
 
-        body = s()
+        body_list = []
 
-        main = wrap_in_block map_process_list_compact main.sexp_body
-        body << main if main
+        main_block = process(main)
+        line = main_block.line
+        body_list << main_block if main_block.sexp_type != :void_stmt
 
-        if rescue_block
-          body.push(*process(rescue_block))
-          body << process(else_block) if else_block
-          body = s(s(:rescue, *body))
-        elsif else_block
-          body << process(else_block)
-        end
+        body_list.push(*process(rescue_block)) if rescue_block
+        body_list << process(else_block) if else_block
+        body_list = [s(:rescue, *body_list)] if rescue_block
 
         if ensure_block
-          body << process(ensure_block)
-          body = s(s(:ensure, *body))
+          body_list << process(ensure_block)
+          body_list = [s(:ensure, *body_list)]
         end
 
-        wrap_in_block(body) || s()
+        wrap_in_block(body_list, line)
       end
 
       def process_rescue_mod(exp)
@@ -138,12 +120,16 @@ module RipperRubyParser
 
       def process_lambda(exp)
         _, args, statements = exp.shift 3
+
         old_type = args.sexp_type
         args = convert_arguments(process(args))
+        statements = process(statements)
+        line = args.line || statements.line
         args = nil if args == s(:args) && old_type == :params
-        make_iter(s(:call, nil, :lambda),
-                  args,
-                  safe_unwrap_void_stmt(process(statements)))
+        call = s(:lambda)
+        call.line = line
+
+        make_iter call, args, safe_unwrap_void_stmt(statements)
       end
 
       private
@@ -165,7 +151,10 @@ module RipperRubyParser
       end
 
       def handle_splat(splat)
-        if splat && splat != 0
+        if splat == 0
+          # Only relevant for Ruby < 2.6
+          [s(:excessed_comma)]
+        elsif splat
           [process(splat)]
         else
           []
@@ -197,17 +186,24 @@ module RipperRubyParser
         [process(block)]
       end
 
-      def convert_empty_to_nil_symbol(block)
-        case block.length
-        when 0
-          s(:nil)
+      def handle_rescue_class_list(eclass)
+        if eclass.nil?
+          s(:array)
+        elsif eclass.first.is_a? Symbol
+          eclass = process(eclass)
+          body = eclass.sexp_body
+          if eclass.sexp_type == :mrhs
+            body.first
+          else
+            s(:array, *body)
+          end
         else
-          block
+          s(:array, process(eclass.first))
         end
       end
 
       def make_iter(call, args, stmt)
-        args.pop if args && args.last == s(:excessed_comma)
+        args[-1] = nil if args && args.last == s(:excessed_comma)
         args ||= 0
         if stmt.empty?
           s(:iter, call, args)
@@ -215,17 +211,6 @@ module RipperRubyParser
           s(:iter, call, args, stmt)
         end
       end
-
-      def wrap_in_block(statements)
-        case statements.length
-        when 0
-          nil
-        when 1
-          statements.first
-        else
-          s(:block, *statements)
-        end
-      end
     end
   end
 end