ripper_ruby_parser 1.7.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 943fa1c037a2a2a257d13aed73d33194756f57d0d582207062331f60b5629cd3
-  data.tar.gz: 94fffed38a9e3354a8f9a903add44e26d5bbc8ea599afb72141fbb877f4e2e22
+  metadata.gz: 5d4b6b4c9b802cf538662716b22a68c98d26af9dd0fba04aa72c53515d869738
+  data.tar.gz: e34339faf0080e288368618bd0688525e253300c25883fc74b8f86adfd572f9b
 SHA512:
-  metadata.gz: b677389067d2759571fb431a8055a4909b58026c127b43fde51d158e6de48633f423cb88f448a996da0406e2e02ff60e9834a25aeff2c5eb49274e80d63d6558
-  data.tar.gz: 47df66fb3e972978919e3224522957ef7ccc3a430ee09318775efd1191c1d87aaf54432aca181ea36b4abc729847eda1355809e5d785a6e68ea1c2ae15c96968
+  metadata.gz: 1262aa2a0cb8b14e99650b8e49f1f1cf4ba672c59ed662b46204c0fcbd1d83580681d3b16440047b29dbc542a8c3dc2bd794071d32e09290a51a4fa4c6180476
+  data.tar.gz: aedb6d0c231afd60183a0cf1152f34c5cc14ec088a22360d63ae4d3aacb50d2e8bc07548e8617611086ed1eddfa1ebb4c6d9aeb12f2093ad6d677e86e2ee363f

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 1.7.2 / 2020-02-28
+
+### Bug fixes
+
+* Support imaginary number literals ([#100])
+* Handle anonymous kwrest arguments ([#95])
+
+### Internal changes
+
+* Update tests ([#101])
+* Prepare for testing on Ruby 2.7 ([#99])
+* Various improvements ([#98])
+* Split long module ([#97])
+
 ## 1.7.1 / 2019-11-03
 
 * Handle unicode escapes with five or six hex digits ([#94])
@@ -187,6 +201,12 @@
 * Initial release
 
 <!-- Pull request links -->
+[#101]: https://github.com/mvz/ripper_ruby_parser/pull/101
+[#100]: https://github.com/mvz/ripper_ruby_parser/pull/100
+[#99]: https://github.com/mvz/ripper_ruby_parser/pull/99
+[#98]: https://github.com/mvz/ripper_ruby_parser/pull/98
+[#97]: https://github.com/mvz/ripper_ruby_parser/pull/97
+[#95]: https://github.com/mvz/ripper_ruby_parser/pull/95
 [#94]: https://github.com/mvz/ripper_ruby_parser/pull/94
 [#92]: https://github.com/mvz/ripper_ruby_parser/pull/92
 [#90]: https://github.com/mvz/ripper_ruby_parser/pull/90

data/lib/ripper_ruby_parser/sexp_handlers.rb

@@ -10,6 +10,7 @@ require "ripper_ruby_parser/sexp_handlers/loops"
 require "ripper_ruby_parser/sexp_handlers/method_calls"
 require "ripper_ruby_parser/sexp_handlers/methods"
 require "ripper_ruby_parser/sexp_handlers/operators"
+require "ripper_ruby_parser/sexp_handlers/string_literals"
 
 module RipperRubyParser
   # Umbrella module for handlers of particular sexp types
@@ -28,6 +29,7 @@ module RipperRubyParser
         include MethodCalls
         include Methods
         include Operators
+        include StringLiterals
       end
     end
   end

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -35,7 +35,7 @@ module RipperRubyParser
 
       def process_kwrest_param(exp)
         _, sym, = exp.shift 3
-        process(sym)
+        process(sym) || s(:lvar, :"")
       end
 
       def process_block_var(exp)
@@ -189,7 +189,12 @@ module RipperRubyParser
       def handle_double_splat(doublesplat)
         return [] unless doublesplat
 
-        [s(:dsplat, process(doublesplat))]
+        # Anonymous kwrest arguments are parsed into an Integer in Ruby 2.4
+        if RUBY_VERSION < "2.5.0" && doublesplat.is_a?(Integer)
+          [s(:dsplat, s(:lvar, :""))]
+        else
+          [s(:dsplat, process(doublesplat))]
+        end
       end
 
       def handle_block_argument(block)

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -4,134 +4,10 @@ module RipperRubyParser
   module SexpHandlers
     # Sexp handlers for literals
     module Literals
-      def process_string_literal(exp)
-        _, content = exp.shift 2
-        process(content)
-      end
-
-      def process_string_content(exp)
-        _, *rest = shift_all exp
-        line, string, rest = extract_string_parts(rest)
-
-        if rest.empty?
-          with_line_number(line, s(:str, string))
-        else
-          s(:dstr, string, *rest)
-        end
-      end
-
-      alias process_word process_string_content
-
-      def process_string_embexpr(exp)
-        _, list = exp.shift 2
-
-        val = process(list.sexp_body.first)
-
-        case val.sexp_type
-        when :str, :dstr
-          val
-        when :void_stmt
-          s(:dstr, "", s(:evstr))
-        else
-          s(:dstr, "", s(:evstr, val))
-        end
-      end
-
-      def process_string_dvar(exp)
-        _, list = exp.shift 2
-        val = process(list)
-        s(:dstr, "", s(:evstr, val))
-      end
-
-      def process_string_concat(exp)
-        _, left, right = exp.shift 3
-
-        left = process(left)
-        right = process(right)
-
-        if left.sexp_type == :str
-          merge_left_into_right(left, right)
-        else
-          merge_right_into_left(left, right)
-        end
-      end
-
-      def process_xstring_literal(exp)
-        _, content = exp.shift 2
-        process(content)
-      end
-
-      def process_xstring(exp)
-        _, *rest = shift_all exp
-        line, string, rest = extract_string_parts(rest)
-        if rest.empty?
-          s(:xstr, string).line(line)
-        else
-          s(:dxstr, string, *rest)
-        end
-      end
-
-      def process_regexp_literal(exp)
-        _, content, (_, flags,) = exp.shift 3
-
-        content = process(content)
-        numflags = character_flags_to_numerical flags
-
-        if content.length == 2
-          return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
-        end
-
-        content.sexp_type = :dregx_once if /o/.match?(flags)
-        content << numflags unless numflags == 0
-        content
-      end
-
-      def process_regexp(exp)
-        _, *rest = shift_all exp
-        line, string, rest = extract_string_parts(rest)
-        with_line_number(line, s(:dregx, string, *rest))
-      end
-
-      def process_symbol_literal(exp)
-        _, symbol = exp.shift 2
-        handle_symbol_content(symbol)
-      end
-
-      def process_symbol(exp)
-        _, node = exp.shift 2
-        handle_symbol_content(node)
-      end
-
-      def process_dyna_symbol(exp)
-        _, node = exp.shift 2
-        handle_dyna_symbol_content(node)
-      end
-
-      def process_qsymbols(exp)
-        _, *items = shift_all(exp)
-        items = items.map { |item| handle_symbol_content(item) }
-        s(:qsymbols, *items)
-      end
-
-      def process_symbols(exp)
-        _, *items = shift_all(exp)
-        items = items.map { |item| handle_dyna_symbol_content(item) }
-        s(:symbols, *items)
-      end
-
-      INTERPOLATING_HEREDOC = /^<<[-~]?[^-~']/.freeze
-      NON_INTERPOLATING_HEREDOC = /^<<[-~]?'/.freeze
-      INTERPOLATING_STRINGS = ['"', "`", ':"', /^%Q.$/, /^%.$/].freeze
-      NON_INTERPOLATING_STRINGS = ["'", ":'", /^%q.$/].freeze
-      INTERPOLATING_WORD_LIST = /^%[WI].$/.freeze
-      NON_INTERPOLATING_WORD_LIST = /^%[wi].$/.freeze
-      REGEXP_LITERALS = ["/", /^%r.$/].freeze
-
-      def process_at_tstring_content(exp)
-        _, content, pos, delim = exp.shift 4
-        string = handle_string_unescaping(content, delim)
-        string = handle_string_encoding(string, delim)
-        with_position(pos, s(:str, string))
+      # character literals
+      def process_at_CHAR(exp)
+        _, val, pos = exp.shift 3
+        with_position(pos, s(:str, unescape(val[1..-1])))
       end
 
       def process_array(exp)
@@ -163,134 +39,31 @@ module RipperRubyParser
         s(:kwsplat, process(param))
       end
 
-      private
-
-      def extract_string_parts(list)
-        return nil, "", [] if list.empty?
-
-        list = merge_raw_string_literals list
-        list = map_process_list list
-
-        parts = list.flat_map do |item|
-          type, val, *rest = item
-          if type == :dstr
-            if val.empty?
-              rest
-            else
-              [s(:str, val), *rest]
-            end
-          else
-            [item]
-          end
-        end
-
-        string = ""
-        while parts.first&.sexp_type == :str
-          str = parts.shift
-          line ||= str.line
-          string += str.last
-        end
-
-        return line, string, parts
-      end
-
-      def merge_raw_string_literals(list)
-        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
-        chunks.flat_map do |is_simple, items|
-          if is_simple && items.count > 1
-            head = items.first
-            contents = items.map { |it| it[1] }.join
-            [s(:@tstring_content, contents, head[2], head[3])]
-          else
-            items
-          end
-        end
-      end
-
-      def character_flags_to_numerical(flags)
-        numflags = 0
-
-        numflags = Regexp::MULTILINE if /m/.match?(flags)
-        numflags |= Regexp::EXTENDED if /x/.match?(flags)
-        numflags |= Regexp::IGNORECASE if /i/.match?(flags)
-
-        numflags |= Regexp::NOENCODING if /n/.match?(flags)
-        numflags |= Regexp::FIXEDENCODING if /[ues]/.match?(flags)
-
-        numflags
-      end
-
-      def handle_dyna_symbol_content(node)
-        type, *body = *process(node)
-        case type
-        when :str, :xstr
-          s(:lit, body.first.to_sym)
-        when :dstr, :dxstr
-          s(:dsym, *body)
-        end
+      # number literals
+      def process_at_int(exp)
+        make_literal(exp) { |val| Integer(val) }
       end
 
-      def handle_symbol_content(node)
-        if node.sexp_type == :'@kw'
-          symbol, position = extract_node_symbol_with_position(node)
-          with_position(position, s(:lit, symbol))
-        else
-          processed = process(node)
-          symbol = processed.last.to_sym
-          line = processed.line
-          with_line_number(line, s(:lit, symbol))
-        end
+      def process_at_float(exp)
+        make_literal(exp, &:to_f)
       end
 
-      def merge_left_into_right(left, right)
-        right[1] = left.last + right[1]
-        right
+      def process_at_rational(exp)
+        make_literal(exp, &:to_r)
       end
 
-      def merge_right_into_left(left, right)
-        if right.sexp_type == :str
-          left.push right
-        else
-          _, first, *rest = right
-          left.push s(:str, first) unless first.empty?
-          left.push(*rest)
-        end
+      def process_at_imaginary(exp)
+        make_literal(exp, &:to_c)
       end
 
-      def handle_string_unescaping(content, delim)
-        case delim
-        when INTERPOLATING_HEREDOC
-          unescape(content)
-        when *INTERPOLATING_STRINGS
-          unescape(content)
-        when INTERPOLATING_WORD_LIST
-          unescape_wordlist_word(content)
-        when *NON_INTERPOLATING_STRINGS
-          simple_unescape(content)
-        when *REGEXP_LITERALS
-          unescape_regexp(content)
-        when NON_INTERPOLATING_WORD_LIST
-          simple_unescape_wordlist_word(content)
-        else
-          content
-        end
-      end
-
-      def handle_string_encoding(string, delim)
-        case delim
-        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
-          fix_encoding string
-        else
-          string
-        end
-      end
+      private
 
       # Process list of items that can be either :assoc_new or :assoc_splat
       def make_hash_items(elems)
         result = s()
         elems.each do |sub_exp|
           if sub_exp.sexp_type == :assoc_new
-            sub_exp.sexp_body.each { |elem| result << process(elem) }
+            result += process(sub_exp).sexp_body
           else # :assoc_splat
             result << process(sub_exp)
           end

data/lib/ripper_ruby_parser/sexp_handlers/methods.rb

@@ -99,20 +99,22 @@ module RipperRubyParser
 
       def convert_arguments(args)
         args.line ||= args.sexp_body.first&.line
-        args.map! do |item|
-          if item.is_a? Symbol
-            item
+        args.map! { |item| convert_argument item }
+      end
+
+      def convert_argument(item)
+        if item.is_a? Symbol
+          item
+        else
+          case item.sexp_type
+          when :lvar
+            item.last
+          when *SPECIAL_ARG_MARKER.keys
+            convert_marked_argument(item)
+          when :masgn
+            convert_masgn_argument(item)
           else
-            case item.sexp_type
-            when :lvar
-              item.last
-            when *SPECIAL_ARG_MARKER.keys
-              convert_marked_argument(item)
-            when :masgn
-              convert_masgn_argument(item)
-            else
-              item
-            end
+            item
           end
         end
       end

data/lib/ripper_ruby_parser/sexp_handlers/string_literals.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module RipperRubyParser
+  module SexpHandlers
+    # Sexp handlers for string and stringlike literals
+    module StringLiterals
+      def process_string_literal(exp)
+        _, content = exp.shift 2
+        process(content)
+      end
+
+      def process_string_content(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+
+        if rest.empty?
+          with_line_number(line, s(:str, string))
+        else
+          s(:dstr, string, *rest)
+        end
+      end
+
+      alias process_word process_string_content
+
+      def process_string_embexpr(exp)
+        _, list = exp.shift 2
+
+        val = process(list.sexp_body.first)
+
+        case val.sexp_type
+        when :str, :dstr
+          val
+        when :void_stmt
+          s(:dstr, "", s(:evstr))
+        else
+          s(:dstr, "", s(:evstr, val))
+        end
+      end
+
+      def process_string_dvar(exp)
+        _, list = exp.shift 2
+        val = process(list)
+        s(:dstr, "", s(:evstr, val))
+      end
+
+      def process_string_concat(exp)
+        _, left, right = exp.shift 3
+
+        left = process(left)
+        right = process(right)
+
+        if left.sexp_type == :str
+          merge_left_into_right(left, right)
+        else
+          merge_right_into_left(left, right)
+        end
+      end
+
+      def process_xstring_literal(exp)
+        _, content = exp.shift 2
+        process(content)
+      end
+
+      def process_xstring(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+        result = if rest.empty?
+                   s(:xstr, string)
+                 else
+                   s(:dxstr, string, *rest)
+                 end
+        result.line = line
+        result
+      end
+
+      def process_regexp_literal(exp)
+        _, content, (_, flags,) = exp.shift 3
+
+        content = process(content)
+        numflags = character_flags_to_numerical flags
+
+        if content.length == 2
+          return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
+        end
+
+        content.sexp_type = :dregx_once if /o/.match?(flags)
+        content << numflags unless numflags == 0
+        content
+      end
+
+      def process_regexp(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+        with_line_number(line, s(:dregx, string, *rest))
+      end
+
+      def process_symbol_literal(exp)
+        _, symbol = exp.shift 2
+        handle_symbol_content(symbol)
+      end
+
+      def process_symbol(exp)
+        _, node = exp.shift 2
+        handle_symbol_content(node)
+      end
+
+      def process_dyna_symbol(exp)
+        _, node = exp.shift 2
+        handle_dyna_symbol_content(node)
+      end
+
+      def process_qsymbols(exp)
+        _, *items = shift_all(exp)
+        items = items.map { |item| handle_symbol_content(item) }
+        s(:qsymbols, *items)
+      end
+
+      def process_symbols(exp)
+        _, *items = shift_all(exp)
+        items = items.map { |item| handle_dyna_symbol_content(item) }
+        s(:symbols, *items)
+      end
+
+      def process_at_tstring_content(exp)
+        _, content, pos, delim = exp.shift 4
+        string = handle_string_unescaping(content, delim)
+        string = handle_string_encoding(string, delim)
+        with_position(pos, s(:str, string))
+      end
+
+      private
+
+      def extract_string_parts(list)
+        return nil, "", [] if list.empty?
+
+        list = merge_raw_string_literals list
+        list = map_process_list list
+
+        parts = list.flat_map do |item|
+          type, val, *rest = item
+          if type == :dstr
+            if val.empty?
+              rest
+            else
+              [s(:str, val), *rest]
+            end
+          else
+            [item]
+          end
+        end
+
+        string = ""
+        while parts.first&.sexp_type == :str
+          str = parts.shift
+          line ||= str.line
+          string += str.last
+        end
+
+        return line, string, parts
+      end
+
+      def merge_raw_string_literals(list)
+        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
+        chunks.flat_map do |is_simple, items|
+          if is_simple && items.count > 1
+            head = items.first
+            contents = items.map { |it| it[1] }.join
+            [s(:@tstring_content, contents, head[2], head[3])]
+          else
+            items
+          end
+        end
+      end
+
+      def character_flags_to_numerical(flags)
+        numflags = 0
+
+        numflags = Regexp::MULTILINE if /m/.match?(flags)
+        numflags |= Regexp::EXTENDED if /x/.match?(flags)
+        numflags |= Regexp::IGNORECASE if /i/.match?(flags)
+
+        numflags |= Regexp::NOENCODING if /n/.match?(flags)
+        numflags |= Regexp::FIXEDENCODING if /[ues]/.match?(flags)
+
+        numflags
+      end
+
+      def handle_dyna_symbol_content(node)
+        type, *body = *process(node)
+        case type
+        when :str, :xstr
+          s(:lit, body.first.to_sym)
+        when :dstr, :dxstr
+          s(:dsym, *body)
+        end
+      end
+
+      def handle_symbol_content(node)
+        if node.sexp_type == :'@kw'
+          symbol, position = extract_node_symbol_with_position(node)
+          with_position(position, s(:lit, symbol))
+        else
+          processed = process(node)
+          symbol = processed.last.to_sym
+          line = processed.line
+          with_line_number(line, s(:lit, symbol))
+        end
+      end
+
+      def merge_left_into_right(left, right)
+        right[1] = left.last + right[1]
+        right
+      end
+
+      def merge_right_into_left(left, right)
+        if right.sexp_type == :str
+          left.push right
+        else
+          _, first, *rest = right
+          left.push s(:str, first) unless first.empty?
+          left.push(*rest)
+        end
+      end
+
+      INTERPOLATING_HEREDOC = /^<<[-~]?[^-~']/.freeze
+      NON_INTERPOLATING_HEREDOC = /^<<[-~]?'/.freeze
+      INTERPOLATING_STRINGS = ['"', "`", ':"', /^%Q.$/, /^%.$/].freeze
+      NON_INTERPOLATING_STRINGS = ["'", ":'", /^%q.$/].freeze
+      INTERPOLATING_WORD_LIST = /^%[WI].$/.freeze
+      NON_INTERPOLATING_WORD_LIST = /^%[wi].$/.freeze
+      REGEXP_LITERALS = ["/", /^%r.$/].freeze
+
+      def handle_string_unescaping(content, delim)
+        case delim
+        when INTERPOLATING_HEREDOC, *INTERPOLATING_STRINGS
+          unescape(content)
+        when INTERPOLATING_WORD_LIST
+          unescape_wordlist_word(content)
+        when *NON_INTERPOLATING_STRINGS
+          simple_unescape(content)
+        when *REGEXP_LITERALS
+          unescape_regexp(content)
+        when NON_INTERPOLATING_WORD_LIST
+          simple_unescape_wordlist_word(content)
+        else
+          content
+        end
+      end
+
+      def handle_string_encoding(string, delim)
+        case delim
+        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
+          fix_encoding string
+        else
+          string
+        end
+      end
+    end
+  end
+end