ripper_ruby_parser 1.6.0 → 1.8.0

data/lib/ripper_ruby_parser/sexp_handlers/string_literals.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module RipperRubyParser
+  module SexpHandlers
+    # Sexp handlers for string and stringlike literals
+    module StringLiterals
+      def process_string_literal(exp)
+        _, content = exp.shift 2
+        process(content)
+      end
+
+      def process_string_content(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+
+        if rest.empty?
+          with_line_number(line, s(:str, string))
+        else
+          s(:dstr, string, *rest)
+        end
+      end
+
+      alias process_word process_string_content
+
+      def process_string_embexpr(exp)
+        _, list = exp.shift 2
+
+        val = process(list.sexp_body.first)
+
+        case val.sexp_type
+        when :str, :dstr
+          val
+        when :void_stmt
+          s(:dstr, "", s(:evstr))
+        else
+          s(:dstr, "", s(:evstr, val))
+        end
+      end
+
+      def process_string_dvar(exp)
+        _, list = exp.shift 2
+        val = process(list)
+        s(:dstr, "", s(:evstr, val))
+      end
+
+      def process_string_concat(exp)
+        _, left, right = exp.shift 3
+
+        left = process(left)
+        right = process(right)
+
+        if left.sexp_type == :str
+          merge_left_into_right(left, right)
+        else
+          merge_right_into_left(left, right)
+        end
+      end
+
+      def process_xstring_literal(exp)
+        _, content = exp.shift 2
+        process(content)
+      end
+
+      def process_xstring(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+        result = if rest.empty?
+                   s(:xstr, string)
+                 else
+                   s(:dxstr, string, *rest)
+                 end
+        result.line = line
+        result
+      end
+
+      def process_regexp_literal(exp)
+        _, content, (_, flags,) = exp.shift 3
+
+        content = process(content)
+        numflags = character_flags_to_numerical flags
+
+        if content.length == 2
+          return with_line_number(content.line, s(:lit, Regexp.new(content.last, numflags)))
+        end
+
+        content.sexp_type = :dregx_once if /o/.match?(flags)
+        content << numflags unless numflags == 0
+        content
+      end
+
+      def process_regexp(exp)
+        _, *rest = shift_all exp
+        line, string, rest = extract_string_parts(rest)
+        with_line_number(line, s(:dregx, string, *rest))
+      end
+
+      def process_symbol_literal(exp)
+        _, symbol = exp.shift 2
+        handle_symbol_content(symbol)
+      end
+
+      def process_symbol(exp)
+        _, node = exp.shift 2
+        handle_symbol_content(node)
+      end
+
+      def process_dyna_symbol(exp)
+        _, node = exp.shift 2
+        handle_dyna_symbol_content(node)
+      end
+
+      def process_qsymbols(exp)
+        _, *items = shift_all(exp)
+        items = items.map { |item| handle_symbol_content(item) }
+        s(:qsymbols, *items)
+      end
+
+      def process_symbols(exp)
+        _, *items = shift_all(exp)
+        items = items.map { |item| handle_dyna_symbol_content(item) }
+        s(:symbols, *items)
+      end
+
+      def process_at_tstring_content(exp)
+        _, content, pos, delim = exp.shift 4
+        string = handle_string_unescaping(content, delim)
+        string = handle_string_encoding(string, delim)
+        with_position(pos, s(:str, string))
+      end
+
+      private
+
+      def extract_string_parts(list)
+        return nil, "", [] if list.empty?
+
+        list = merge_raw_string_literals list
+        list = map_process_list list
+        parts = unpack_dstr list
+        merge_initial_string_literals(parts)
+      end
+
+      def merge_raw_string_literals(list)
+        chunks = list.chunk { |it| it.sexp_type == :@tstring_content }
+        chunks.flat_map do |is_simple, items|
+          if is_simple && items.count > 1
+            head = items.first
+            contents = items.map { |it| it[1] }.join
+            [s(:@tstring_content, contents, head[2], head[3])]
+          else
+            items
+          end
+        end
+      end
+
+      def unpack_dstr(list)
+        list.flat_map do |item|
+          type, val, *rest = item
+          if type == :dstr
+            if val.empty?
+              rest
+            else
+              [s(:str, val), *rest]
+            end
+          else
+            [item]
+          end
+        end
+      end
+
+      def merge_initial_string_literals(parts)
+        string = ""
+        while parts.first&.sexp_type == :str
+          str = parts.shift
+          line ||= str.line
+          string += str.last
+        end
+
+        return line, string, parts
+      end
+
+      def character_flags_to_numerical(flags)
+        numflags = 0
+
+        numflags = Regexp::MULTILINE if /m/.match?(flags)
+        numflags |= Regexp::EXTENDED if /x/.match?(flags)
+        numflags |= Regexp::IGNORECASE if /i/.match?(flags)
+
+        numflags |= Regexp::NOENCODING if /n/.match?(flags)
+        numflags |= Regexp::FIXEDENCODING if /[ues]/.match?(flags)
+
+        numflags
+      end
+
+      def handle_dyna_symbol_content(node)
+        type, *body = *process(node)
+        case type
+        when :str, :xstr
+          s(:lit, body.first.to_sym)
+        when :dstr, :dxstr
+          s(:dsym, *body)
+        end
+      end
+
+      def handle_symbol_content(node)
+        if node.sexp_type == :@kw
+          symbol, position = extract_node_symbol_with_position(node)
+          with_position(position, s(:lit, symbol))
+        else
+          processed = process(node)
+          symbol = processed.last.to_sym
+          line = processed.line
+          with_line_number(line, s(:lit, symbol))
+        end
+      end
+
+      def merge_left_into_right(left, right)
+        right[1] = left.last + right[1]
+        right
+      end
+
+      def merge_right_into_left(left, right)
+        if right.sexp_type == :str
+          left.push right
+        else
+          _, first, *rest = right
+          left.push s(:str, first) unless first.empty?
+          left.push(*rest)
+        end
+      end
+
+      INTERPOLATING_HEREDOC = /^<<[-~]?[^-~']/.freeze
+      NON_INTERPOLATING_HEREDOC = /^<<[-~]?'/.freeze
+      INTERPOLATING_STRINGS = ['"', "`", ':"', /^%Q.$/, /^%.$/].freeze
+      NON_INTERPOLATING_STRINGS = ["'", ":'", /^%q.$/].freeze
+      INTERPOLATING_WORD_LIST = /^%[WI].$/.freeze
+      NON_INTERPOLATING_WORD_LIST = /^%[wi].$/.freeze
+      REGEXP_LITERALS = ["/", /^%r.$/].freeze
+
+      def handle_string_unescaping(content, delim)
+        case delim
+        when INTERPOLATING_HEREDOC, *INTERPOLATING_STRINGS
+          unescape(content)
+        when INTERPOLATING_WORD_LIST
+          unescape_wordlist_word(content)
+        when *NON_INTERPOLATING_STRINGS
+          simple_unescape(content)
+        when *REGEXP_LITERALS
+          unescape_regexp(content)
+        when NON_INTERPOLATING_WORD_LIST
+          simple_unescape_wordlist_word(content)
+        else
+          content
+        end
+      end
+
+      def handle_string_encoding(string, delim)
+        case delim
+        when INTERPOLATING_HEREDOC, INTERPOLATING_WORD_LIST, *INTERPOLATING_STRINGS
+          fix_encoding string
+        else
+          string
+        end
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_processor.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'sexp_processor'
-require 'ripper_ruby_parser/sexp_handlers'
-require 'ripper_ruby_parser/unescape'
+require "sexp_processor"
+require "ripper_ruby_parser/sexp_handlers"
+require "ripper_ruby_parser/unescape"
 
 module RipperRubyParser
   # Processes the sexp created by Ripper to what RubyParser would produce.
@@ -11,31 +11,16 @@ module RipperRubyParser
   class SexpProcessor < ::SexpProcessor
     include Unescape
 
-    attr_reader :filename
-    attr_reader :extra_compatible
+    attr_reader :filename, :extra_compatible
 
     def initialize(filename: nil, extra_compatible: nil)
       super()
 
-      @processors[:@int] = :process_at_int
-      @processors[:@float] = :process_at_float
-      @processors[:@rational] = :process_at_rational
-      @processors[:@CHAR] = :process_at_CHAR
-      @processors[:@label] = :process_at_label
-
-      @processors[:@const] = :process_at_const
-      @processors[:@ident] = :process_at_ident
-      @processors[:@cvar] = :process_at_cvar
-      @processors[:@gvar] = :process_at_gvar
-      @processors[:@ivar] = :process_at_ivar
-      @processors[:@kw] = :process_at_kw
-      @processors[:@op] = :process_at_op
-      @processors[:@backref] = :process_at_backref
-
-      @processors[:@backtick] = :process_at_backtick
-      @processors[:@period] = :process_at_period
-
-      @processors[:@tstring_content] = :process_at_tstring_content
+      public_methods.each do |name|
+        if name =~ /^process_at_(.*)/
+          @processors["@#{Regexp.last_match(1)}".to_sym] = name.to_sym
+        end
+      end
 
       @filename = filename
       @extra_compatible = extra_compatible
@@ -45,6 +30,8 @@ module RipperRubyParser
       @in_method_body = false
       @kwrest = []
       @block_kwrest = []
+
+      @kept_comment = nil
     end
 
     include SexpHandlers
@@ -56,37 +43,31 @@ module RipperRubyParser
     end
 
     def process_module(exp)
-      _, const_ref, body = exp.shift 3
-      const, line = const_ref_to_const_with_line_number const_ref
-      with_line_number(line,
-                       s(:module, const, *class_or_module_body(body)))
+      _, const_ref, body, pos = exp.shift 4
+      const = const_ref_to_const const_ref
+      with_position(pos,
+                    s(:module, const, *class_or_module_body(body)))
     end
 
     def process_class(exp)
-      _, const_ref, parent, body = exp.shift 4
-      const, line = const_ref_to_const_with_line_number const_ref
+      _, const_ref, parent, body, pos = exp.shift 5
+      const = const_ref_to_const const_ref
       parent = process(parent)
-      with_line_number(line,
-                       s(:class, const, parent, *class_or_module_body(body)))
+      with_position(pos,
+                    s(:class, const, parent, *class_or_module_body(body)))
     end
 
     def process_sclass(exp)
-      _, klass, block = exp.shift 3
-      s(:sclass, process(klass), *class_or_module_body(block))
+      _, klass, block, pos = exp.shift 4
+      with_position pos, s(:sclass, process(klass), *class_or_module_body(block))
     end
 
     def process_stmts(exp)
       _, *statements = shift_all(exp)
-      statements = map_process_list_compact statements
-      case statements.count
-      when 0
-        s(:void_stmt)
-      when 1
-        statements.first
-      else
-        first = statements.shift
-        s(:block, *unwrap_block(first), *statements)
-      end
+      statements = map_unwrap_begin_list map_process_list statements
+      line = statements.first.line
+      statements = reject_void_stmt statements
+      wrap_in_block(statements, line)
     end
 
     def process_var_ref(exp)
@@ -104,6 +85,11 @@ module RipperRubyParser
       s(:valias, left[1].to_sym, right[1].to_sym)
     end
 
+    def process_void_stmt(exp)
+      _, pos = exp.shift 2
+      with_position pos, s(:void_stmt)
+    end
+
     def process_const_path_ref(exp)
       _, left, right = exp.shift 3
       s(:colon2, process(left), extract_node_symbol(process(right)))
@@ -130,49 +116,33 @@ module RipperRubyParser
     def process_paren(exp)
       _, body = exp.shift 2
       result = process body
-      if result.sexp_type == :void_stmt
-        s(:nil)
-      else
-        result
-      end
+      convert_void_stmt_to_nil_symbol result
     end
 
     def process_comment(exp)
       _, comment, inner = exp.shift 3
+      comment = @kept_comment + comment if @kept_comment
+      @kept_comment = nil
       sexp = process(inner)
-      sexp.comments = comment
+      case sexp.sexp_type
+      when :defs, :defn, :module, :class, :sclass
+        sexp.comments = comment
+      else
+        @kept_comment = comment
+      end
       sexp
     end
 
     def process_BEGIN(exp)
-      _, body = exp.shift 2
+      _, body, pos = exp.shift 3
       body = reject_void_stmt map_process_list body.sexp_body
-      s(:iter, s(:preexe), 0, *body)
+      with_position pos, s(:iter, s(:preexe), 0, *body)
     end
 
     def process_END(exp)
-      _, body = exp.shift 2
+      _, body, pos = exp.shift 3
       body = map_process_list_compact body.sexp_body
-      s(:iter, s(:postexe), 0, *body)
-    end
-
-    # number literals
-    def process_at_int(exp)
-      make_literal(exp) { |val| Integer(val) }
-    end
-
-    def process_at_float(exp)
-      make_literal(exp, &:to_f)
-    end
-
-    def process_at_rational(exp)
-      make_literal(exp, &:to_r)
-    end
-
-    # character literals
-    def process_at_CHAR(exp)
-      _, val, pos = exp.shift 3
-      with_position(pos, s(:str, unescape(val[1..-1])))
+      with_position pos, s(:iter, s(:postexe), 0, *body)
     end
 
     def process_at_label(exp)
@@ -227,7 +197,7 @@ module RipperRubyParser
       _, str, pos = exp.shift 3
       name = str[1..-1]
       with_position pos do
-        if name =~ /[0-9]/
+        if /[0-9]/.match?(name)
           s(:nth_ref, name.to_i)
         else
           s(:back_ref, name.to_sym)
@@ -242,17 +212,16 @@ module RipperRubyParser
 
     private
 
-    def const_ref_to_const_with_line_number(const_ref)
+    def const_ref_to_const(const_ref)
       const = process(const_ref)
-      line = const.line
       const = const[1] if const.sexp_type == :const
-      return const, line
+      const
     end
 
     def class_or_module_body(exp)
       body = process(exp)
 
-      return body if body.empty?
+      return [] if body.sexp_type == :void_stmt
 
       unwrap_block body
     end