ripper_ruby_parser 1.7.2 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d4b6b4c9b802cf538662716b22a68c98d26af9dd0fba04aa72c53515d869738
-  data.tar.gz: e34339faf0080e288368618bd0688525e253300c25883fc74b8f86adfd572f9b
+  metadata.gz: a110a987032924880be79cfbb79948e14f90868e8105055a0833284a6f939731
+  data.tar.gz: f38c7fa39cb0f1f6fc78cbe72956c77fe15fe74fe99507a3bd59d057618aed0a
 SHA512:
-  metadata.gz: 1262aa2a0cb8b14e99650b8e49f1f1cf4ba672c59ed662b46204c0fcbd1d83580681d3b16440047b29dbc542a8c3dc2bd794071d32e09290a51a4fa4c6180476
-  data.tar.gz: aedb6d0c231afd60183a0cf1152f34c5cc14ec088a22360d63ae4d3aacb50d2e8bc07548e8617611086ed1eddfa1ebb4c6d9aeb12f2093ad6d677e86e2ee363f
+  metadata.gz: 7bf9631eb8941e2ed07ec265519ce8860f9d417cb224b2bbd1bbb8b32277374d76936be126c7893f73bcbcaa1414a25d4ca2280c9e0e701462cdbf5d34c3c77f
+  data.tar.gz: e517b02e7854fc644763932d8935dd5b145e2c7c4546874b419932a1d0f0d53b523cba356949f5b941e1fe6405c5b308cade8c42ba3d02078a8e7abe5df176a6

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+Notable changes to this project will be documented in this file.
+
+This project adheres to [Semantic Versioning 2.0.0][1].
+
+This document is formatted based on [Keep A CHANGELOG][2].
+
+## 1.10.0 / 2022-03-13
+
+* Handle shadow arguments ([#161])
+* Drop support for Ruby 2.5 ([#165])
+* Support running on Ruby 3.1([#180])
+
+## 1.9.0 / 2021-08-10
+
+* Fix escape sequence handling in non-interpolating strings and word lists
+  ([#152])
+* Handle strings with mixed valid and invalid utf8 ([#153])
+* Handle argument forwarding, updating the RubyParser compatibility target to
+  3.17.0 ([#154])
+* Fix encoding for non-interpolating strings as well ([#155])
+
+## 1.8.0 / 2021-05-18
+
+### Changes
+
+* Drop support for Ruby 2.4 ([2cf6b4050] and [#145])
+* Update RubyParser compatibility target to 3.15.0 ([#105])
+* Fix handling of dedented heredoc ([#106])
+* Add support for running on Ruby 3.0 ([#123])
+* Various code improvements and other internal changes
+
 ## 1.7.2 / 2020-02-28
 
 ### Bug fixes
@@ -201,6 +232,17 @@
 * Initial release
 
 <!-- Pull request links -->
+[#180]: https://github.com/mvz/ripper_ruby_parser/pull/180
+[#165]: https://github.com/mvz/ripper_ruby_parser/pull/165
+[#161]: https://github.com/mvz/ripper_ruby_parser/pull/161
+[#155]: https://github.com/mvz/ripper_ruby_parser/pull/155
+[#154]: https://github.com/mvz/ripper_ruby_parser/pull/154
+[#153]: https://github.com/mvz/ripper_ruby_parser/pull/153
+[#152]: https://github.com/mvz/ripper_ruby_parser/pull/152
+[#145]: https://github.com/mvz/ripper_ruby_parser/pull/145
+[#123]: https://github.com/mvz/ripper_ruby_parser/pull/123
+[#106]: https://github.com/mvz/ripper_ruby_parser/pull/106
+[#105]: https://github.com/mvz/ripper_ruby_parser/pull/105
 [#101]: https://github.com/mvz/ripper_ruby_parser/pull/101
 [#100]: https://github.com/mvz/ripper_ruby_parser/pull/100
 [#99]: https://github.com/mvz/ripper_ruby_parser/pull/99
@@ -231,3 +273,11 @@
 [#67]: https://github.com/mvz/ripper_ruby_parser/pull/67
 [#66]: https://github.com/mvz/ripper_ruby_parser/pull/66
 [#65]: https://github.com/mvz/ripper_ruby_parser/pull/65
+
+<!-- Git sha links -->
+[2cf6b4050]: https://github.com/mvz/ripper_ruby_parser/commit/2cf6b40501576003b916196e23222276b51aa53c
+
+<!-- Other links -->
+
+[1]:  https://semver.org/spec/v2.0.0.html
+[2]:  https://keepachangelog.com/en/1.0.0/

data/README.md

@@ -10,8 +10,8 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 
 * Drop-in replacement for RubyParser
 * Should handle 1.9 and later syntax gracefully
-* Requires Ruby 2.4 or higher
-* Compatible with RubyParser 3.14.0
+* Requires Ruby 2.6 or higher
+* Compatible with RubyParser 3.18.0
 
 ## Known incompatibilities
 
@@ -44,7 +44,7 @@ parser.parse "foo[bar] += baz qux"
 
 ## Requirements
 
-* Ruby 2.4 or higher
+* Ruby 2.5 or higher
 * `sexp_processor`
 
 ## Hacking and contributing
@@ -64,7 +64,7 @@ If you want to send pull requests or patches, please:
 
 (The MIT License)
 
-Copyright (c) 2012, 2014-2019 Matijs van Zuijlen
+Copyright (c) 2012, 2014-2022 Matijs van Zuijlen
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -27,6 +27,8 @@ module RipperRubyParser
       Sexp.from_array(result)
     end
 
+    private
+
     def on_backtick(delimiter)
       @delimiter_stack.push delimiter
       super
@@ -101,6 +103,18 @@ module RipperRubyParser
       @delimiter_stack.push delimiter
     end
 
+    def on_heredoc_dedent(val, width)
+      next_dedent = true
+      val.map! do |e|
+        if e.is_a?(Array) && e[0] == :@tstring_content
+          e = dedent_element(e, width) if next_dedent
+          next_dedent = e[1].end_with? "\n"
+        end
+        e
+      end
+      val
+    end
+
     def on_heredoc_end(_delimiter)
       @delimiter_stack.pop
     end
@@ -293,28 +307,26 @@ module RipperRubyParser
       commentize(:END, super)
     end
 
-    def on_parse_error(*args)
-      raise SyntaxError, *args
+    def on_parse_error(message)
+      raise SyntaxError, message
     end
 
-    def on_class_name_error(*args)
-      raise SyntaxError, *args
+    def on_class_name_error(message, *)
+      raise SyntaxError, message
     end
 
-    def on_alias_error(*args)
-      raise SyntaxError, *args
+    def on_alias_error(message, *)
+      raise SyntaxError, message
     end
 
-    def on_assign_error(*args)
-      raise SyntaxError, *args
+    def on_assign_error(message, *)
+      raise SyntaxError, message
     end
 
-    def on_param_error(*args)
-      raise SyntaxError, *args
+    def on_param_error(message, *)
+      raise SyntaxError, message
     end
 
-    private
-
     def commentize(_name, exp)
       (_, _kw, loc), comment = @comment_stack.pop
       @comment = ""

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -118,8 +118,8 @@ module RipperRubyParser
       end
 
       OPERATOR_ASSIGNMENT_MAP = {
-        '||': :op_asgn_or,
-        '&&': :op_asgn_and
+        "||": :op_asgn_or,
+        "&&": :op_asgn_and
       }.freeze
 
       def create_operator_assignment_sub_type(lvalue, value, operator)

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -14,16 +14,20 @@ module RipperRubyParser
         make_iter call, args, safe_unwrap_void_stmt(stmt)
       end
 
+      # NOTE: Argument forwarding is handled differently in Ruby 3.0 and 3.1
+      # 3.0: s(:params, nil, nil, s(:args_forward), nil, nil, nil, nil)
+      # 3.1: s(:params, nil, nil, nil, nil, nil, s(:args_forward), :&)
       def process_params(exp)
         _, normal, defaults, splat, rest, kwargs, doublesplat, block = exp.shift 8
 
-        args = handle_normal_arguments normal
-        args += handle_default_arguments defaults
-        args += handle_splat splat
-        args += handle_normal_arguments rest
-        args += handle_kwargs kwargs
-        args += handle_double_splat doublesplat
-        args += handle_block_argument block
+        args =
+          handle_normal_arguments(normal) +
+          handle_default_arguments(defaults) +
+          handle_splat(splat) +
+          handle_normal_arguments(rest) +
+          handle_kwargs(kwargs) +
+          handle_double_splat(doublesplat) +
+          handle_block_argument(block)
 
         s(:args, *args)
       end
@@ -39,17 +43,22 @@ module RipperRubyParser
       end
 
       def process_block_var(exp)
-        _, args, = exp.shift 3
+        _, args, shadowargs = exp.shift 3
 
         names = process(args)
 
+        if shadowargs
+          shadowargs = map_process_list(shadowargs).map { |item| item[1] }
+          names << s(:shadow, *shadowargs)
+        end
+
         convert_arguments names
       end
 
       def process_begin(exp)
         _, body, pos = exp.shift 3
 
-        body = convert_empty_to_nil_symbol process(body)
+        body = convert_void_stmt_to_nil_symbol process(body)
         with_position pos, s(:begin, body)
       end
 
@@ -58,19 +67,7 @@ module RipperRubyParser
         rescue_block = map_process_list_compact block.sexp_body
         rescue_block << nil if rescue_block.empty?
 
-        capture = if eclass.nil?
-                    s(:array)
-                  elsif eclass.first.is_a? Symbol
-                    eclass = process(eclass)
-                    body = eclass.sexp_body
-                    if eclass.sexp_type == :mrhs
-                      body.first
-                    else
-                      s(:array, *body)
-                    end
-                  else
-                    s(:array, process(eclass.first))
-                  end
+        capture = handle_rescue_class_list eclass
 
         capture << create_assignment_sub_type(process(evar), s(:gvar, :$!)) if evar
 
@@ -80,23 +77,22 @@ module RipperRubyParser
       def process_bodystmt(exp)
         _, main, rescue_block, else_block, ensure_block = exp.shift 5
 
-        body = s()
+        body_list = []
 
-        main_list = map_unwrap_begin_list map_process_list main.sexp_body
-        line = main_list.first.line
-        main = wrap_in_block reject_void_stmt main_list
-        body << main if main
+        main_block = process(main)
+        line = main_block.line
+        body_list << main_block if main_block.sexp_type != :void_stmt
 
-        body.push(*process(rescue_block)) if rescue_block
-        body << process(else_block) if else_block
-        body = s(s(:rescue, *body)) if rescue_block
+        body_list.push(*process(rescue_block)) if rescue_block
+        body_list << process(else_block) if else_block
+        body_list = [s(:rescue, *body_list)] if rescue_block
 
         if ensure_block
-          body << process(ensure_block)
-          body = s(s(:ensure, *body))
+          body_list << process(ensure_block)
+          body_list = [s(:ensure, *body_list)]
         end
 
-        wrap_in_block(body) || s().line(line)
+        wrap_in_block(body_list, line)
       end
 
       def process_rescue_mod(exp)
@@ -189,26 +185,35 @@ module RipperRubyParser
       def handle_double_splat(doublesplat)
         return [] unless doublesplat
 
-        # Anonymous kwrest arguments are parsed into an Integer in Ruby 2.4
-        if RUBY_VERSION < "2.5.0" && doublesplat.is_a?(Integer)
-          [s(:dsplat, s(:lvar, :""))]
+        contents = process(doublesplat)
+        case contents.sexp_type
+        when :forward_args # Argument forwarding in Ruby 3.1
+          [contents]
         else
-          [s(:dsplat, process(doublesplat))]
+          [s(:dsplat, contents)]
         end
       end
 
       def handle_block_argument(block)
         return [] unless block
+        return [] if block == :& # Part of argument forwarding in Ruby 3.1; ignore
 
         [process(block)]
       end
 
-      def convert_empty_to_nil_symbol(block)
-        case block.length
-        when 0
-          s(:nil)
+      def handle_rescue_class_list(eclass)
+        if eclass.nil?
+          s(:array)
+        elsif eclass.first.is_a? Symbol
+          eclass = process(eclass)
+          body = eclass.sexp_body
+          if eclass.sexp_type == :mrhs
+            body.first
+          else
+            s(:array, *body)
+          end
         else
-          block
+          s(:array, process(eclass.first))
         end
       end
 
@@ -221,17 +226,6 @@ module RipperRubyParser
           s(:iter, call, args, stmt)
         end
       end
-
-      def wrap_in_block(statements)
-        case statements.length
-        when 0
-          nil
-        when 1
-          statements.first
-        else
-          s(:block, *statements)
-        end
-      end
     end
   end
 end

data/lib/ripper_ruby_parser/sexp_handlers/conditionals.rb

@@ -47,33 +47,20 @@ module RipperRubyParser
 
       def process_case(exp)
         _, expr, clauses = exp.shift 3
-        s(:case, process(expr), *process(clauses))
+        s(:case, process(expr), *process(clauses).sexp_body)
       end
 
       def process_when(exp)
         _, values, truepart, falsepart = exp.shift 4
 
-        falsepart = process(falsepart)
-        falsepart = unwrap_nil falsepart if falsepart
-
-        if falsepart.nil?
-          falsepart = [nil]
-        elsif falsepart.first.is_a? Symbol
-          falsepart = s(falsepart)
-        end
+        falsepart ||= s(:void_stmt)
 
+        falsepart = unwrap_case_body process(falsepart)
         values = process(values).sexp_body
+        truepart = unwrap_block process(truepart)
 
-        truepart = map_process_list_compact truepart.sexp_body
-        truepart = if truepart.empty?
-                     [nil]
-                   else
-                     unwrap_block(truepart.shift) + truepart
-                   end
-
-        s(s(:when,
-            s(:array, *values),
-            *truepart),
+        s(:case_body,
+          s(:when, s(:array, *values), *truepart),
           *falsepart)
       end
 
@@ -109,6 +96,17 @@ module RipperRubyParser
           s(:if, cond, truepart, falsepart)
         end
       end
+
+      def unwrap_case_body(exp)
+        case exp.sexp_type
+        when :case_body
+          exp.sexp_body
+        when :void_stmt
+          [nil]
+        else
+          [exp]
+        end
+      end
     end
   end
 end

data/lib/ripper_ruby_parser/sexp_handlers/helper_methods.rb

@@ -81,13 +81,46 @@ module RipperRubyParser
       end
 
       def unwrap_block(exp)
-        if exp.sexp_type == :block
+        case exp.sexp_type
+        when :block
           exp.sexp_body
+        when :void_stmt
+          [nil]
         else
           [exp]
         end
       end
 
+      def wrap_in_block(statements, line)
+        case statements.length
+        when 0
+          s(:void_stmt).line(line)
+        when 1
+          statements.first
+        else
+          first = statements.shift
+          s(:block, *unwrap_block(first), *statements)
+        end
+      end
+
+      def convert_void_stmt_to_nil_symbol(block)
+        case block.sexp_type
+        when :void_stmt
+          s(:nil)
+        else
+          block
+        end
+      end
+
+      def convert_empty_to_nil_symbol(block)
+        case block.length
+        when 0
+          s(:nil)
+        else
+          block
+        end
+      end
+
       def handle_return_argument_list(arglist)
         args = process(arglist).sexp_body
 

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -7,7 +7,7 @@ module RipperRubyParser
       # character literals
       def process_at_CHAR(exp)
         _, val, pos = exp.shift 3
-        with_position(pos, s(:str, unescape(val[1..-1])))
+        with_position(pos, s(:str, fix_encoding(unescape(val[1..]))))
       end
 
       def process_array(exp)

data/lib/ripper_ruby_parser/sexp_handlers/method_calls.rb

@@ -17,7 +17,7 @@ module RipperRubyParser
 
       def process_arg_paren(exp)
         _, args = exp.shift 2
-        return s() if args.nil?
+        return s(:arglist) if args.nil?
 
         process(args)
       end
@@ -26,7 +26,11 @@ module RipperRubyParser
         _, call, parens = exp.shift 3
         call = process(call)
         parens = process(parens)
-        call.push(*parens.sexp_body)
+        if parens.sexp_type == :forward_args
+          call.push(parens)
+        else
+          call.push(*parens.sexp_body)
+        end
       end
 
       # Handle implied hashes, such as at the end of argument lists.
@@ -36,9 +40,9 @@ module RipperRubyParser
       end
 
       CALL_OP_MAP = {
-        '.': :call,
-        '::': :call,
-        '&.': :safe_call
+        ".": :call,
+        "::": :call,
+        "&.": :safe_call
       }.freeze
 
       def process_call(exp)

data/lib/ripper_ruby_parser/sexp_handlers/methods.rb

@@ -72,6 +72,11 @@ module RipperRubyParser
         s(:alias, process(left), process(right))
       end
 
+      def process_args_forward(exp)
+        _ = exp.shift
+        s(:forward_args)
+      end
+
       private
 
       def in_method
@@ -83,8 +88,8 @@ module RipperRubyParser
 
       def method_body(exp)
         block = process exp
-        case block.length
-        when 0
+        case block.sexp_type
+        when :void_stmt
           [s(:nil).line(block.line)]
         else
           unwrap_block block
@@ -99,23 +104,20 @@ module RipperRubyParser
 
       def convert_arguments(args)
         args.line ||= args.sexp_body.first&.line
-        args.map! { |item| convert_argument item }
+        args.sexp_body = args.sexp_body.map { |item| convert_argument item }
+        args
       end
 
       def convert_argument(item)
-        if item.is_a? Symbol
-          item
+        case item.sexp_type
+        when :lvar
+          item.last
+        when *SPECIAL_ARG_MARKER.keys
+          convert_marked_argument(item)
+        when :masgn
+          convert_masgn_argument(item)
         else
-          case item.sexp_type
-          when :lvar
-            item.last
-          when *SPECIAL_ARG_MARKER.keys
-            convert_marked_argument(item)
-          when :masgn
-            convert_masgn_argument(item)
-          else
-            item
-          end
+          item
         end
       end
 

data/lib/ripper_ruby_parser/sexp_handlers/operators.rb

@@ -5,8 +5,8 @@ module RipperRubyParser
     # Sexp handlers for operators
     module Operators
       BINARY_OPERATOR_MAP = {
-        '&&': :and,
-        '||': :or,
+        "&&": :and,
+        "||": :or,
         and: :and,
         or: :or
       }.freeze
@@ -16,7 +16,7 @@ module RipperRubyParser
       }.freeze
 
       NEGATED_BINARY_OPERATOR_MAP = {
-        '!~': :=~
+        "!~": :=~
       }.freeze
 
       SHIFT_OPERATORS = [:<<, :>>].freeze

data/lib/ripper_ruby_parser/sexp_handlers/string_literals.rb

@@ -123,8 +123,7 @@ module RipperRubyParser
 
       def process_at_tstring_content(exp)
         _, content, pos, delim = exp.shift 4
-        string = handle_string_unescaping(content, delim)
-        string = handle_string_encoding(string, delim)
+        string = fix_encoding handle_string_unescaping(content, delim)
         with_position(pos, s(:str, string))
       end
 
@@ -135,8 +134,25 @@ module RipperRubyParser
 
         list = merge_raw_string_literals list
         list = map_process_list list
+        parts = unpack_dstr list
+        merge_initial_string_literals(parts)
+      end
 
-        parts = list.flat_map do |item|
+      def merge_raw_string_literals(list)
+        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
+        chunks.flat_map do |is_simple, items|
+          if is_simple && items.count > 1
+            head = items.first
+            contents = items.map { |it| it[1] }.join
+            [s(:@tstring_content, contents, head[2], head[3])]
+          else
+            items
+          end
+        end
+      end
+
+      def unpack_dstr(list)
+        list.flat_map do |item|
           type, val, *rest = item
           if type == :dstr
             if val.empty?
@@ -148,7 +164,9 @@ module RipperRubyParser
             [item]
           end
         end
+      end
 
+      def merge_initial_string_literals(parts)
         string = ""
         while parts.first&.sexp_type == :str
           str = parts.shift
@@ -159,19 +177,6 @@ module RipperRubyParser
         return line, string, parts
       end
 
-      def merge_raw_string_literals(list)
-        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
-        chunks.flat_map do |is_simple, items|
-          if is_simple && items.count > 1
-            head = items.first
-            contents = items.map { |it| it[1] }.join
-            [s(:@tstring_content, contents, head[2], head[3])]
-          else
-            items
-          end
-        end
-      end
-
       def character_flags_to_numerical(flags)
         numflags = 0
 
@@ -196,7 +201,7 @@ module RipperRubyParser
       end
 
       def handle_symbol_content(node)
-        if node.sexp_type == :'@kw'
+        if node.sexp_type == :@kw
           symbol, position = extract_node_symbol_with_position(node)
           with_position(position, s(:lit, symbol))
         else
@@ -237,24 +242,15 @@ module RipperRubyParser
         when INTERPOLATING_WORD_LIST
           unescape_wordlist_word(content)
         when *NON_INTERPOLATING_STRINGS
-          simple_unescape(content)
+          simple_unescape(content, delim)
         when *REGEXP_LITERALS
           unescape_regexp(content)
         when NON_INTERPOLATING_WORD_LIST
-          simple_unescape_wordlist_word(content)
+          simple_unescape_wordlist_word(content, delim)
         else
           content
         end
       end
-
-      def handle_string_encoding(string, delim)
-        case delim
-        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
-          fix_encoding string
-        else
-          string
-        end
-      end
     end
   end
 end