ripper_ruby_parser 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab036613dacd6865ca8103db78c67273d6fad340e3a6a7ed1fe5450c5511cc05
-  data.tar.gz: 182feec3370738d6d5516e5615022a045ed717edf489cc694ac02eda97e6e6bb
+  metadata.gz: 05e260a0b747a456d0a25ce09e6607bcaf29fcfef68eec466c87e502ad27d435
+  data.tar.gz: 77a0985075b11dd8461dd96a4b1def05c34bee338713701cfc2fa0fa026414bd
 SHA512:
-  metadata.gz: 9c5f17c00265aaf9df0741c5afee3b90f000141d4285bce067953bb1feac645779c8e4b9db2319baa9a3f6822b46a9f6edbeef87af1177b22a03cd437fecad51
-  data.tar.gz: 7001633d0756ab5afb2b2813572550f3cea1df75c11e82fe8117e4243aaa6cf4386512b3a7754b76f0a5a8be7c4b49f8b1e5a0148b4465e7aa72658b7c951acc
+  metadata.gz: a9d499d3e3d60b5d758884e0f4c3667e19c58943f0662aa0936e8188ba6a8767b844734a930e32e7485196abc5e050986f27cd491829500c639562de3d1e54e9
+  data.tar.gz: 8476d87b6a80339014fc7e47a1d795ce6554d9f89650a272ea7040a7e84922b491ef0dd48a2fa858d436b3538d5c670624303a5f9a11b4f501fc3351a3d6aa15

data/CHANGELOG.md

@@ -1,12 +1,43 @@
 # Changelog
 
+## 1.5.0 / 2019-03-18
+
+* Process embedded documents as comments
+* Handle \u{xxxx} form of unicode escape sequence
+* Treat keyword rest arguments (double spats) as local variables, for blocks
+  and for methods.
+  Versions of Ripper before CRuby 2.6 do not handle these correctly, so
+  RipperRubyParser adds its own accounting of these parameters.
+* Do not unescape heredocs with single quotes. These are heredocs where the
+  marker is a single-quoted string
+* Increase compatibility when handling a begin..end block, e.g., as an assigned
+  value, as a receiver, as a condition
+* Fix handling of for loops that assign to multiple loop variables
+* Handle aliasing for method names that are keywords, e.g., `alias next succ`
+* Do not crash on rational literals
+* Restore `#extra_compatible` flag
+* Match RubyParser bug in handling rescue modifier when combined with
+  assignment. See RubyParser
+  [issue #227](https://github.com/seattlerb/ruby_parser/issues/227).
+  This behavior is only enabled when `#extra_compatible` is set to true
+* Fix compatibility when assigning to a class variable inside the method
+  argument specification
+* Match RubyParser's handling of byte sequences in second and further literal
+  parts of strings with interpolations. This behavior is only enabled when
+  `#extra_compatible` is set to true
+* Require Ruby 2.3 or higher
+* Mark as compatible with RubyParser 3.12.0
+* Match RubyParser's handling of block keyword rest arguments.
+  This behavior is only enabled when `#extra_compatible` is set to true
+* Support Ruby 2.6
+
 ## 1.4.2 / 2018-04-03
 
 * Fix handling of strings delimited by %()
 * Handle line continuations in stringlike literals
   - Handle line continuations in string and regexp literals
   - Handle escaped line continuations
-  - Hanlde line continuations in word and symbol list literals
+  - Handle line continuations in word and symbol list literals
 * Force encoding of string literals to UTF-8 if the result is valid
 * Fix handling of range operators with float literals
 
@@ -40,6 +71,7 @@
 * Improve handling of boolean operators with parenthes
 * Improve compatibility for begin..end blocks used as method and operator
   arguments.
+* Support Ruby 2.5
 * Drop support for Ruby 2.0 and 2.1
 * Handle `__ENCODING__` constant.
 

data/README.md

@@ -2,35 +2,67 @@
 
 by Matijs van Zuijlen
 
-http://www.github.com/mvz/ripper_ruby_parser
-
 ## Description
 
 Parse with Ripper, produce sexps that are compatible with RubyParser.
 
 ## Features/Notes
 
-* Drop-in replacement for RubyParser.
-* Should handle 1.9 and later syntax gracefully.
-* Requires MRI 2.2 or higher
+* Drop-in replacement for RubyParser
+* Should handle 1.9 and later syntax gracefully
+* Requires MRI 2.3 or higher
+* Compatible with RubyParser 3.12.0
+
+## Known incompatibilities
+
+RipperRubyParser has some incompatibilities with RubyParser. For some of these,
+the behavior can be changed by turning on extra-compatible mode.
+
+The following incompatibilities cannot be changed:
+
+* RipperRubyParser won't handle non-UTF-8 files without an encoding comment,
+  just like regular Ruby
+* RipperRubyParser keeps carriage return characters in heredocs that include them
+* RipperRubyParser does not attempt to match RubyParser's line numbering bugs
+
+The following incompatibilities can be made compatible by turning on
+extra-compatible mode:
+
+* RipperRubyParser handles unicode escapes without braces correctly, while
+  RubyParser absorbs trailing hexadecimal characters
+* RipperRubyParser handles the rescue modifier correctly, while RubyParser
+  still contains a bug that was fixed in Ruby 2.4. See RubyParser
+  [issue #227](https://github.com/seattlerb/ruby_parser/issues/227).
+* RubyParser handles byte sequences in second and further literal parts of a
+  strings with interpolations differently. RipperRubyParser will convert these
+  to unicode if possible.
+* RubyParser handles byte sequences in heredocs and interpolating word lists
+  differently. RipperRubyParser will convert these to unicode if possible.
 
 ## Install
 
-* gem install ripper_ruby_parser
+    gem install ripper_ruby_parser
 
 ## Synopsis
 
     require 'ripper_ruby_parser'
 
     parser = RipperRubyParser::Parser.new
-    result = parser.parse "puts 'Hello World'"
-    p result
+    parser.parse "puts 'Hello World'"
     # => s(:call, nil, :puts, s(:arglist, s(:str, "Hello World!")))
 
+    parser.parse '"foo\u273bbar"'
+    # => s(:str, "foo✻bar")
+
+    parser.extra_compatible = true
+
+    parser.parse '"foo\u273bbar"'
+    # => s(:str, "foo✻r")
+
 ## Requirements
 
 * Ruby 2.2 or higher
-* sexp_processor
+* `sexp_processor`
 
 ## Hacking and contributing
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rake/clean'
 require 'bundler/gem_tasks'
 require 'rake/testtask'

data/lib/ripper_ruby_parser.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 require 'ripper_ruby_parser/version'
 require 'ripper_ruby_parser/parser'

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ripper'
 require 'ripper_ruby_parser/syntax_error'
 require 'ripper_ruby_parser/unescape'
@@ -10,14 +12,16 @@ module RipperRubyParser
   class CommentingRipperParser < Ripper::SexpBuilder
     def initialize(*args)
       super
-      @comment = nil
+      @comment = ''
       @comment_stack = []
       @delimiter_stack = []
+      @space_before = false
+      @seen_space = false
       @in_symbol = false
     end
 
     def parse
-      result = suppress_warnings { super }
+      result = super
       raise 'Ripper parse failed.' unless result
 
       Sexp.from_array(result)
@@ -25,13 +29,22 @@ module RipperRubyParser
 
     def on_backtick(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_comment(tok)
-      @comment ||= ''
       @comment += tok
-      super
+    end
+
+    def on_embdoc_beg(tok)
+      @comment += tok
+    end
+
+    def on_embdoc(tok)
+      @comment += tok
+    end
+
+    def on_embdoc_end(tok)
+      @comment += tok
     end
 
     def on_kw(tok)
@@ -39,7 +52,7 @@ module RipperRubyParser
       when 'class', 'def', 'module'
         unless @in_symbol
           @comment_stack.push [tok.to_sym, @comment]
-          @comment = nil
+          @comment = ''
         end
       end
       super
@@ -75,12 +88,10 @@ module RipperRubyParser
 
     def on_heredoc_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
-    def on_heredoc_end(delimiter)
+    def on_heredoc_end(_delimiter)
       @delimiter_stack.pop
-      super
     end
 
     def on_mlhs_new
@@ -105,7 +116,6 @@ module RipperRubyParser
 
     def on_qsymbols_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_qsymbols_new
@@ -118,7 +128,6 @@ module RipperRubyParser
 
     def on_qwords_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_qwords_new
@@ -131,7 +140,6 @@ module RipperRubyParser
 
     def on_regexp_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_regexp_end(delimiter)
@@ -161,7 +169,6 @@ module RipperRubyParser
 
     def on_symbols_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_symbols_new
@@ -174,27 +181,10 @@ module RipperRubyParser
 
     def on_tstring_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_tstring_content(content)
-      content = case @delimiter_stack.last
-                when /^<</
-                  Unescape.unescape(content)
-                when '"', '`', ':"', /^%Q.$/, /^%.$/
-                  Unescape.fix_encoding(Unescape.unescape(content))
-                when /^%[WI].$/
-                  Unescape.fix_encoding(Unescape.unescape_wordlist_word(content))
-                when "'", ":'", /^%q.$/
-                  Unescape.simple_unescape(content)
-                when '/', /^%r.$/
-                  Unescape.unescape_regexp(content)
-                when /^%[wi].$/
-                  Unescape.simple_unescape_wordlist_word(content)
-                else
-                  content
-                end
-      super(content)
+      super(content) << @delimiter_stack.last
     end
 
     def on_tstring_end(delimiter)
@@ -212,7 +202,6 @@ module RipperRubyParser
 
     def on_words_beg(delimiter)
       @delimiter_stack.push delimiter
-      super
     end
 
     def on_words_new
@@ -238,7 +227,6 @@ module RipperRubyParser
 
     def on_sp(_token)
       @seen_space = true
-      super
     end
 
     def on_int(_token)
@@ -269,7 +257,6 @@ module RipperRubyParser
     def on_symbeg(delimiter)
       @delimiter_stack.push delimiter
       @in_symbol = true
-      super
     end
 
     def on_symbol(*args)
@@ -280,7 +267,6 @@ module RipperRubyParser
 
     def on_embexpr_beg(_delimiter)
       @in_symbol = false
-      super
     end
 
     def on_dyna_symbol(*args)
@@ -312,16 +298,8 @@ module RipperRubyParser
 
     def commentize(_name, exp)
       _tok, comment = @comment_stack.pop
-      @comment = nil
-      [:comment, comment || '', exp]
-    end
-
-    def suppress_warnings
-      old_verbose = $VERBOSE
-      $VERBOSE = nil
-      result = yield
-      $VERBOSE = old_verbose
-      result
+      @comment = ''
+      [:comment, comment, exp]
     end
   end
 end

data/lib/ripper_ruby_parser/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ripper_ruby_parser/commenting_ripper_parser'
 require 'ripper_ruby_parser/sexp_processor'
 
@@ -5,13 +7,21 @@ module RipperRubyParser
   # Main parser class. Brings together Ripper and our
   # RipperRubyParser::SexpProcessor.
   class Parser
+    attr_accessor :extra_compatible
+
+    def initialize
+      @extra_compatible = false
+    end
+
     def parse(source, filename = '(string)', lineno = 1)
       parser = CommentingRipperParser.new(source, filename, lineno)
       exp = parser.parse
 
-      processor = SexpProcessor.new(filename: filename)
+      processor = SexpProcessor.new(filename: filename, extra_compatible: extra_compatible)
       result = processor.process exp
 
+      result = result[1] if result.sexp_type == :begin
+
       if result.sexp_type == :void_stmt
         nil
       else

data/lib/ripper_ruby_parser/sexp_handlers.rb

@@ -1,11 +1,10 @@
+# frozen_string_literal: true
+
 require 'ripper_ruby_parser/sexp_handlers/helper_methods'
 
-require 'ripper_ruby_parser/sexp_handlers/arguments'
-require 'ripper_ruby_parser/sexp_handlers/arrays'
 require 'ripper_ruby_parser/sexp_handlers/assignment'
 require 'ripper_ruby_parser/sexp_handlers/blocks'
 require 'ripper_ruby_parser/sexp_handlers/conditionals'
-require 'ripper_ruby_parser/sexp_handlers/hashes'
 require 'ripper_ruby_parser/sexp_handlers/literals'
 require 'ripper_ruby_parser/sexp_handlers/loops'
 require 'ripper_ruby_parser/sexp_handlers/method_calls'
@@ -21,12 +20,9 @@ module RipperRubyParser
       base.class_eval do
         include HelperMethods
 
-        include Arguments
-        include Arrays
         include Assignment
         include Blocks
         include Conditionals
-        include Hashes
         include Literals
         include Loops
         include MethodCalls

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -1,17 +1,27 @@
+# frozen_string_literal: true
+
 module RipperRubyParser
   module SexpHandlers
     # Sexp handlers for assignments
     module Assignment
       def process_assign(exp)
         _, lvalue, value = exp.shift 3
+        if extra_compatible && value.sexp_type == :rescue_mod
+          if [:command, :command_call].include? value[1].sexp_type
+            return process s(:rescue_mod, s(:assign, lvalue, value[1]), value[2])
+          end
+        end
+
         lvalue = process(lvalue)
         value = process(value)
 
         case value.sexp_type
         when :mrhs
           value.sexp_type = :svalue
-        when :fake_array
+        when :args
           value = s(:svalue, s(:array, *value.sexp_body))
+        else
+          value = unwrap_begin(value)
         end
 
         with_line_number(lvalue.line,
@@ -21,30 +31,26 @@ module RipperRubyParser
       def process_massign(exp)
         _, left, right = exp.shift 3
 
-        left = process left
-
-        left = left[1] if left.sexp_type == :masgn
-        left = create_multiple_assignment_sub_types left.sexp_body
-
+        left = process(left).last
         right = process(right)
 
         case right.sexp_type
-        when :fake_array
-          right[0] = :array
+        when :args
+          right.sexp_type = :array
         when :mrhs
-          right = right[1]
+          _, right = right
         else
           right = s(:to_ary, right)
         end
 
-        s(:masgn, s(:array, *left), right)
+        s(:masgn, left, right)
       end
 
       def process_mrhs_new_from_args(exp)
         _, inner, last = exp.shift 3
-        inner = map_process_sexp_body_compact(inner)
-        inner.push process(last) if last
-        s(:fake_array, *inner)
+        process(inner).tap do |result|
+          result.push process(last) if last
+        end
       end
 
       def process_mrhs_add_star(exp)
@@ -52,34 +58,47 @@ module RipperRubyParser
       end
 
       def process_mlhs_add_star(exp)
-        _, args, splatarg = exp.shift 3
-        items = process args
-        items << s(:splat, process(splatarg))
+        _, base, splatarg = exp.shift 3
+        masgn = process base
+
+        splat = process(splatarg)
+        splat_item = if splat.nil?
+                       s(:splat)
+                     else
+                       s(:splat, create_valueless_assignment_sub_type(splat))
+                     end
+
+        masgn.last << splat_item
+        masgn
       end
 
       def process_mlhs_add_post(exp)
         _, base, rest = exp.shift 3
-        process(base).push(*process(rest).sexp_body)
+        base = process(base)
+        rest = process(rest)
+        base.last.push(*rest.last.sexp_body)
+        base
       end
 
       def process_mlhs_paren(exp)
         _, contents = exp.shift 2
 
-        items = process(contents)
+        process(contents)
+      end
 
-        if items.sexp_type == :mlhs
-          s(:masgn, s(:array, *create_multiple_assignment_sub_types(items.sexp_body)))
-        else
-          items
-        end
+      def process_mlhs(exp)
+        _, *rest = shift_all exp
+
+        items = map_process_list(rest)
+        s(:masgn, s(:array, *create_multiple_assignment_sub_types(items)))
       end
 
       def process_opassign(exp)
-        _, lvalue, operator, value = exp.shift 4
+        _, lvalue, (_, operator,), value = exp.shift 4
 
         lvalue = process(lvalue)
         value = process(value)
-        operator = operator[1].delete('=').to_sym
+        operator = operator.chop.to_sym
 
         create_operator_assignment_sub_type lvalue, value, operator
       end
@@ -88,15 +107,7 @@ module RipperRubyParser
 
       def create_multiple_assignment_sub_types(sexp_list)
         sexp_list.map! do |item|
-          if item.sexp_type == :splat
-            if item[1].nil?
-              s(:splat)
-            else
-              s(:splat, create_valueless_assignment_sub_type(item[1]))
-            end
-          else
-            create_valueless_assignment_sub_type item
-          end
+          create_valueless_assignment_sub_type item
         end
       end
 
@@ -116,11 +127,13 @@ module RipperRubyParser
         case lvalue.sexp_type
         when :aref_field
           _, arr, arglist = lvalue
+          arglist.sexp_type = :arglist
           s(:op_asgn1, arr, arglist, operator, value)
         when :field
           _, obj, _, (_, field) = lvalue
           s(:op_asgn2, obj, :"#{field}=", operator, value)
         else
+          value = unwrap_begin(value)
           if (mapped = OPERATOR_ASSIGNMENT_MAP[operator])
             s(mapped, lvalue, create_assignment_sub_type(lvalue, value))
           else
@@ -158,7 +171,8 @@ module RipperRubyParser
       }.freeze
 
       def create_assignment_sub_type(lvalue, value)
-        s(map_assignment_lvalue_type(lvalue.sexp_type), lvalue[1], value)
+        lvalue_type, lvalue_value = lvalue
+        s(map_assignment_lvalue_type(lvalue_type), lvalue_value, value)
       end
 
       def map_assignment_lvalue_type(type)