ripper_ruby_parser 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bad890447c64972884e94b2470b0fcb8114aad1af6e9b44c27d05f1187a789d3
-  data.tar.gz: ea2965906b883f6b8f1d7481cd04d89c3d1dcc57c8bba022adfc27b1af01df1a
+  metadata.gz: cc516bb8bd2fd0f9fd310f848dbf272481d6c15a8ee3aeecba41ac8e24a7b8c3
+  data.tar.gz: 2d79ba2d094e3694993f48e92565c7424121331aad653bd6cb5fad51a7f9ea75
 SHA512:
-  metadata.gz: c4ef6f13a7914774863c9503dfd27dd099e5adf1f122c7f7f38105b9b7570a7dfe5b51ac744a2a7a200b6c7012937477a933f2651b7e48e7b41b70a81f6bb7dc
-  data.tar.gz: c28f34964b94a8d7bd28fb47924d8ffd7f99104bb22ce90de48ed6b9ac2c1b32b789bf5c5059bc02c7d29ae5d98622cfc72f0d41d6eba6fc77f177e65679917e
+  metadata.gz: 2d521a088a674f96b3639d624de7ababb6dbea93965fee1185b6a14f8a289ef88566a87cc5918004d3765f19de7ea0a15d99cfb30792ff519e1d8426555bb162
+  data.tar.gz: 0a79e0832ce26282bacb13d534dde0460f33b677e7a07c23bde4d530848228bda65460ce2c229a97ddc25c5f12cc0aa75ea176f09e3959fcbf78e34536fa6bcd

data/CHANGELOG.md

@@ -1,9 +1,34 @@
 # Changelog
 
+## 1.6.0 / 2019-04-12
+
+* Fix line numbering for range literals ([#79])
+* Match handling of carriage returns in heredocs in extra-compatible mode
+  ([#77], [#78])
+* Match RubyParser behavior for current Ruby rather than latest Ruby ([#76])
+  - Adjust integration tests to compare against `RubyParser.for_current_ruby`
+  - Remove extra-compatible handling of `rescue` modifier
+* Handle directly nested parentheses in destructuring ([#75])
+* Make results compatible with RubyParser 3.13.1 ([#74])
+  - Remove extra-compatible handling of string literals
+* Improve compatibility for when clauses containing `begin..end` blocks ([#73])
+* Handle use of backtick as a symbol ([#72])
+* Improve string handling compatibility ([#71])
+  - Interpolation with `__FILE__` keyword
+  - Line continuation after interpolation for indentable heredocs
+  - Nested interpolations
+* Handle method argument destructuring ([#70])
+* Improve compatibility of operator assignment ([#69])
+* Handle multiple assignment with RHS that is a block ([#68])
+* Improve compatibility of handling escaped line-endings ([#67])
+* Make results compatible with RubyParser 3.13.0 ([#65])
+  - Change result for `BEGIN { foo }`
+  - Remove extra-compatible handling of rescue modifier
+
 ## 1.5.1 / 2019-03-21
 
 * Fix handling of singleton methods whose names are keywords
-  ([#66](https://github.com/mvz/ripper_ruby_parser/pull/66))
+  ([#66])
 
 ## 1.5.0 / 2019-03-18
 
@@ -131,3 +156,20 @@
 ## 0.0.1 / 2012-03-11
 
 * Initial release
+
+<!-- Pull request links -->
+[#79]: https://github.com/mvz/ripper_ruby_parser/pull/79
+[#78]: https://github.com/mvz/ripper_ruby_parser/pull/78
+[#77]: https://github.com/mvz/ripper_ruby_parser/pull/77
+[#76]: https://github.com/mvz/ripper_ruby_parser/pull/76
+[#75]: https://github.com/mvz/ripper_ruby_parser/pull/75
+[#74]: https://github.com/mvz/ripper_ruby_parser/pull/74
+[#73]: https://github.com/mvz/ripper_ruby_parser/pull/73
+[#72]: https://github.com/mvz/ripper_ruby_parser/pull/72
+[#71]: https://github.com/mvz/ripper_ruby_parser/pull/71
+[#70]: https://github.com/mvz/ripper_ruby_parser/pull/70
+[#69]: https://github.com/mvz/ripper_ruby_parser/pull/69
+[#68]: https://github.com/mvz/ripper_ruby_parser/pull/68
+[#67]: https://github.com/mvz/ripper_ruby_parser/pull/67
+[#66]: https://github.com/mvz/ripper_ruby_parser/pull/66
+[#65]: https://github.com/mvz/ripper_ruby_parser/pull/65

data/README.md

@@ -11,7 +11,7 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 * Drop-in replacement for RubyParser
 * Should handle 1.9 and later syntax gracefully
 * Requires MRI 2.3 or higher
-* Compatible with RubyParser 3.12.0
+* Compatible with RubyParser 3.13.1
 
 ## Known incompatibilities
 
@@ -22,22 +22,16 @@ The following incompatibilities cannot be changed:
 
 * RipperRubyParser won't handle non-UTF-8 files without an encoding comment,
   just like regular Ruby
-* RipperRubyParser keeps carriage return characters in heredocs that include them
 * RipperRubyParser does not attempt to match RubyParser's line numbering bugs
+* RipperRubyParser correctly dedents heredocs with interpolations
 
 The following incompatibilities can be made compatible by turning on
 extra-compatible mode:
 
-* RipperRubyParser handles unicode escapes without braces correctly, while
-  RubyParser absorbs trailing hexadecimal characters
-* RipperRubyParser handles the rescue modifier correctly, while RubyParser
-  still contains a bug that was fixed in Ruby 2.4. See RubyParser
-  [issue #227](https://github.com/seattlerb/ruby_parser/issues/227).
-* RubyParser handles byte sequences in second and further literal parts of a
-  strings with interpolations differently. RipperRubyParser will convert these
-  to unicode if possible.
-* RubyParser handles byte sequences in heredocs and interpolating word lists
-  differently. RipperRubyParser will convert these to unicode if possible.
+* Operator assignment of a method call without parentheses to a collection
+  element uses and `:array` S-expression instead of `:arglist`
+* RipperRubyParser keeps carriage return characters in heredocs that include
+  them
 
 ## Install
 
@@ -45,23 +39,29 @@ extra-compatible mode:
 
 ## Synopsis
 
-    require 'ripper_ruby_parser'
-
-    parser = RipperRubyParser::Parser.new
-    parser.parse "puts 'Hello World'"
-    # => s(:call, nil, :puts, s(:arglist, s(:str, "Hello World!")))
-
-    parser.parse '"foo\u273bbar"'
-    # => s(:str, "foo✻bar")
-
-    parser.extra_compatible = true
-
-    parser.parse '"foo\u273bbar"'
-    # => s(:str, "foo✻r")
+```ruby
+require 'ripper_ruby_parser'
+
+parser = RipperRubyParser::Parser.new
+parser.parse "puts 'Hello World'"
+# => s(:call, nil, :puts, s(:str, "Hello World!"))
+parser.parse "foo[bar] += baz qux"
+# => s(:op_asgn1, s(:call, nil, :foo),
+#      s(:arglist, s(:call, nil, :bar)),
+#      :+,
+#      s(:call, nil, :baz, s(:call, nil, :qux)))
+parser.extra_compatible = true
+
+parser.parse "foo[bar] += baz qux"
+# => s(:op_asgn1, s(:call, nil, :foo),
+#      s(:array, s(:call, nil, :bar)),
+#      :+,
+#      s(:call, nil, :baz, s(:call, nil, :qux)))
+```
 
 ## Requirements
 
-* Ruby 2.2 or higher
+* Ruby 2.3 or higher
 * `sexp_processor`
 
 ## Hacking and contributing
@@ -81,7 +81,7 @@ If you want to send pull requests or patches, please:
 
 (The MIT License)
 
-Copyright (c) 2012, 2014-2018 Matijs van Zuijlen
+Copyright (c) 2012, 2014-2019 Matijs van Zuijlen
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -29,6 +29,7 @@ module RipperRubyParser
 
     def on_backtick(delimiter)
       @delimiter_stack.push delimiter
+      super
     end
 
     def on_comment(tok)

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -6,11 +6,6 @@ module RipperRubyParser
     module Assignment
       def process_assign(exp)
         _, lvalue, value = exp.shift 3
-        if extra_compatible && value.sexp_type == :rescue_mod
-          if [:command, :command_call].include? value[1].sexp_type
-            return process s(:rescue_mod, s(:assign, lvalue, value[1]), value[2])
-          end
-        end
 
         lvalue = process(lvalue)
         value = process(value)
@@ -40,7 +35,7 @@ module RipperRubyParser
         when :mrhs
           _, right = right
         else
-          right = s(:to_ary, right)
+          right = s(:to_ary, unwrap_begin(right))
         end
 
         s(:masgn, left, right)
@@ -83,7 +78,11 @@ module RipperRubyParser
       def process_mlhs_paren(exp)
         _, contents = exp.shift 2
 
-        process(contents)
+        if contents.sexp_type == :mlhs_paren
+          s(:masgn, s(:array, process(contents)))
+        else
+          process(contents)
+        end
       end
 
       def process_mlhs(exp)
@@ -97,10 +96,11 @@ module RipperRubyParser
         _, lvalue, (_, operator,), value = exp.shift 4
 
         lvalue = process(lvalue)
+        original_value_type = value.sexp_type
         value = process(value)
         operator = operator.chop.to_sym
 
-        create_operator_assignment_sub_type lvalue, value, operator
+        create_operator_assignment_sub_type lvalue, value, operator, original_value_type
       end
 
       private
@@ -123,11 +123,16 @@ module RipperRubyParser
         '&&': :op_asgn_and
       }.freeze
 
-      def create_operator_assignment_sub_type(lvalue, value, operator)
+      def create_operator_assignment_sub_type(lvalue, value, operator, original_value_type)
         case lvalue.sexp_type
         when :aref_field
           _, arr, arglist = lvalue
-          arglist.sexp_type = :arglist
+          arglist.sexp_type = if extra_compatible &&
+                                 [:command, :command_call].include?(original_value_type)
+                                :array
+                              else
+                                :arglist
+                              end
           s(:op_asgn1, arr, arglist, operator, value)
         when :field
           _, obj, _, (_, field) = lvalue

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -10,7 +10,7 @@ module RipperRubyParser
         call = process(call)
         args = process(args)
         kwrest = kwrest_param(args) if args
-        stmt = with_block_kwrest(kwrest) { process(stmt) }
+        stmt = with_kwrest(kwrest) { process(stmt) }
         make_iter call, args, safe_unwrap_void_stmt(stmt)
       end
 
@@ -43,7 +43,7 @@ module RipperRubyParser
 
         names = process(args)
 
-        convert_block_args names
+        convert_arguments names
       end
 
       def process_begin(exp)
@@ -107,6 +107,7 @@ module RipperRubyParser
 
       def process_rescue_mod(exp)
         _, scary, safe = exp.shift 3
+
         s(:rescue, process(scary), s(:resbody, s(:array), process(safe)))
       end
 
@@ -138,7 +139,7 @@ module RipperRubyParser
       def process_lambda(exp)
         _, args, statements = exp.shift 3
         old_type = args.sexp_type
-        args = convert_method_args(process(args))
+        args = convert_arguments(process(args))
         args = nil if args == s(:args) && old_type == :params
         make_iter(s(:call, nil, :lambda),
                   args,
@@ -225,35 +226,6 @@ module RipperRubyParser
           s(:block, *statements)
         end
       end
-
-      def convert_block_args(args)
-        args.map! do |item|
-          if item.is_a? Symbol
-            item
-          else
-            case item.sexp_type
-            when :lvar
-              item.last
-            when :masgn
-              args = item[1]
-              args.shift
-              s(:masgn, *convert_block_args(args))
-            when :lasgn
-              if item.length == 2
-                item[1]
-              else
-                item
-              end
-            when *Methods::SPECIAL_ARG_MARKER.keys
-              marker = Methods::SPECIAL_ARG_MARKER[item.sexp_type]
-              name = extract_node_symbol item[1]
-              :"#{marker}#{name}"
-            else
-              item
-            end
-          end
-        end
-      end
     end
   end
 end

data/lib/ripper_ruby_parser/sexp_handlers/conditionals.rb

@@ -65,7 +65,11 @@ module RipperRubyParser
         values = process(values).sexp_body
 
         truepart = map_process_list_compact truepart.sexp_body
-        truepart = [nil] if truepart.empty?
+        truepart = if truepart.empty?
+                     [nil]
+                   else
+                     unwrap_block(truepart.shift) + truepart
+                   end
 
         s(s(:when,
             s(:array, *values),

data/lib/ripper_ruby_parser/sexp_handlers/helper_methods.rb

@@ -80,6 +80,14 @@ module RipperRubyParser
         end
       end
 
+      def unwrap_block(exp)
+        if exp.sexp_type == :block
+          exp.sexp_body
+        else
+          [exp]
+        end
+      end
+
       def handle_return_argument_list(arglist)
         args = process(arglist).sexp_body
 

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -28,19 +28,19 @@ module RipperRubyParser
         val = process(list.sexp_body.first)
 
         case val.sexp_type
-        when :str
+        when :str, :dstr
           val
         when :void_stmt
-          s(:dstr, s(:evstr))
+          s(:dstr, '', s(:evstr))
         else
-          s(:dstr, s(:evstr, val))
+          s(:dstr, '', s(:evstr, val))
         end
       end
 
       def process_string_dvar(exp)
         _, list = exp.shift 2
         val = process(list)
-        s(:dstr, s(:evstr, val))
+        s(:dstr, '', s(:evstr, val))
       end
 
       def process_string_concat(exp)
@@ -164,40 +164,44 @@ module RipperRubyParser
       private
 
       def extract_string_parts(list)
-        parts = []
-
-        unless list.empty?
-          parts << process(list.shift)
-          list.each do |item|
-            parts << if extra_compatible && item.sexp_type == :@tstring_content
-                       alternative_process_at_tstring_content(item)
-                     else
-                       process(item)
-                     end
+        return '', [] if list.empty?
+
+        list = merge_raw_string_literals list
+        list = map_process_list list
+
+        parts = list.flat_map do |item|
+          type, val, *rest = item
+          if type == :dstr
+            if val.empty?
+              rest
+            else
+              [s(:str, val), *rest]
+            end
+          else
+            [item]
           end
         end
 
         string = ''
-        while !parts.empty? && parts.first.sexp_type == :str
+        while parts.first&.sexp_type == :str
           str = parts.shift
           string += str.last
         end
 
-        rest = parts.map { |se| se.sexp_type == :dstr ? se.last : se }
-
-        return string, rest
+        return string, parts
       end
 
-      def alternative_process_at_tstring_content(exp)
-        _, content, _, delim = exp.shift 4
-        string = case delim
-                 when *INTERPOLATING_STRINGS
-                   unescape(content)
-                 else
-                   content
-                 end
-        string.force_encoding('ascii-8bit') if string == "\0"
-        s(:str, string)
+      def merge_raw_string_literals(list)
+        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
+        chunks.flat_map do |is_simple, items|
+          if is_simple && items.count > 1
+            head = items.first
+            contents = items.map { |it| it[1] }.join
+            [s(:@tstring_content, contents, head[2], head[3])]
+          else
+            items
+          end
+        end
       end
 
       def character_flags_to_numerical(flags)
@@ -251,7 +255,13 @@ module RipperRubyParser
 
       def handle_string_unescaping(content, delim)
         case delim
-        when INTERPOLATING_HEREDOC, *INTERPOLATING_STRINGS
+        when INTERPOLATING_HEREDOC
+          if extra_compatible
+            unescape(content).delete("\r")
+          else
+            unescape(content)
+          end
+        when *INTERPOLATING_STRINGS
           unescape(content)
         when INTERPOLATING_WORD_LIST
           unescape_wordlist_word(content)
@@ -268,13 +278,7 @@ module RipperRubyParser
 
       def handle_string_encoding(string, delim)
         case delim
-        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST
-          if extra_compatible
-            string
-          else
-            fix_encoding string
-          end
-        when *INTERPOLATING_STRINGS
+        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
           fix_encoding string
         else
           string