irb 1.6.4 → 1.7.1

data/lib/irb/ruby-lex.rb

@@ -6,6 +6,7 @@
 
 require "ripper"
 require "jruby" if RUBY_ENGINE == "jruby"
+require_relative "nesting_parser"
 
 # :stopdoc:
 class RubyLex
@@ -18,10 +19,7 @@ class RubyLex
 
   def initialize(context)
     @context = context
-    @exp_line_no = @line_no = 1
-    @indent = 0
-    @continue = false
-    @line = ""
+    @line_no = 1
     @prompt = nil
   end
 
@@ -42,14 +40,22 @@ class RubyLex
     result
   end
 
+  def single_line_command?(code)
+    command = code.split(/\s/, 2).first
+    @context.symbol_alias?(command) || @context.transform_args?(command)
+  end
+
   # io functions
-  def set_input(io, &block)
+  def set_input(&block)
+    @input = block
+  end
+
+  def configure_io(io)
     @io = io
     if @io.respond_to?(:check_termination)
       @io.check_termination do |code|
         if Reline::IOGate.in_pasting?
-          lex = RubyLex.new(@context)
-          rest = lex.check_termination_in_prev_line(code)
+          rest = check_termination_in_prev_line(code)
           if rest
             Reline.delete_text
             rest.bytes.reverse_each do |c|
@@ -61,61 +67,39 @@ class RubyLex
           end
         else
           # Accept any single-line input for symbol aliases or commands that transform args
-          command = code.split(/\s/, 2).first
-          if @context.symbol_alias?(command) || @context.transform_args?(command)
-            next true
-          end
+          next true if single_line_command?(code)
 
-          code.gsub!(/\s*\z/, '').concat("\n")
-          tokens = self.class.ripper_lex_without_warning(code, context: @context)
-          ltype, indent, continue, code_block_open = check_state(code, tokens)
-          if ltype or indent > 0 or continue or code_block_open
-            false
-          else
-            true
-          end
+          _tokens, _opens, terminated = check_code_state(code)
+          terminated
         end
       end
     end
     if @io.respond_to?(:dynamic_prompt)
       @io.dynamic_prompt do |lines|
         lines << '' if lines.empty?
-        result = []
         tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
-        code = String.new
-        partial_tokens = []
-        unprocessed_tokens = []
-        line_num_offset = 0
-        tokens.each do |t|
-          partial_tokens << t
-          unprocessed_tokens << t
-          if t.tok.include?("\n")
-            t_str = t.tok
-            t_str.each_line("\n") do |s|
-              code << s
-              next unless s.include?("\n")
-              ltype, indent, continue, code_block_open = check_state(code, partial_tokens)
-              result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
-              line_num_offset += 1
-            end
-            unprocessed_tokens = []
-          else
-            code << t.tok
+        line_results = IRB::NestingParser.parse_by_line(tokens)
+        tokens_until_line = []
+        line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset|
+          line_tokens.each do |token, _s|
+            # Avoid appending duplicated token. Tokens that include "\n" like multiline tstring_content can exist in multiple lines.
+            tokens_until_line << token if token != tokens_until_line.last
           end
+          continue = should_continue?(tokens_until_line)
+          prompt(next_opens, continue, line_num_offset)
         end
-
-        unless unprocessed_tokens.empty?
-          ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens)
-          result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
-        end
-        result
       end
     end
 
-    if block_given?
-      @input = block
-    else
-      @input = Proc.new{@io.gets}
+    if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
+      @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
+        next nil if lines == [nil] # Workaround for exit IRB with CTRL+d
+        next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/)
+
+        code = lines[0..line_index].map { |l| "#{l}\n" }.join
+        tokens = self.class.ripper_lex_without_warning(code, context: @context)
+        process_indent_level(tokens, lines, line_index, is_newline)
+      end
     end
   end
 
@@ -136,9 +120,42 @@ class RubyLex
     "#{local_variables.join('=')}=nil;" unless local_variables.empty?
   end
 
+  # Some part of the code is not included in Ripper's token.
+  # Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr.
+  # With interpolated tokens, tokens.map(&:tok).join will be equal to code.
+  def self.interpolate_ripper_ignored_tokens(code, tokens)
+    line_positions = [0]
+    code.lines.each do |line|
+      line_positions << line_positions.last + line.bytesize
+    end
+    prev_byte_pos = 0
+    interpolated = []
+    prev_line = 1
+    tokens.each do |t|
+      line, col = t.pos
+      byte_pos = line_positions[line - 1] + col
+      if prev_byte_pos < byte_pos
+        tok = code.byteslice(prev_byte_pos...byte_pos)
+        pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
+        interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
+        prev_line += tok.count("\n")
+      end
+      interpolated << t
+      prev_byte_pos = byte_pos + t.tok.bytesize
+      prev_line += t.tok.count("\n")
+    end
+    if prev_byte_pos < code.bytesize
+      tok = code.byteslice(prev_byte_pos..)
+      pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
+      interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0)
+    end
+    interpolated
+  end
+
   def self.ripper_lex_without_warning(code, context: nil)
     verbose, $VERBOSE = $VERBOSE, nil
     lvars_code = generate_local_variables_assign_code(context&.local_variables || [])
+    original_code = code
     if lvars_code
       code = "#{lvars_code}\n#{code}"
       line_no = 0
@@ -148,7 +165,8 @@ class RubyLex
 
     compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no|
       lexer = Ripper::Lexer.new(inner_code, '-', line_no)
-      lexer.scan.each_with_object([]) do |t, tokens|
+      tokens = []
+      lexer.scan.each do |t|
         next if t.pos.first == 0
         prev_tk = tokens.last
         position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize
@@ -158,151 +176,112 @@ class RubyLex
           tokens << t
         end
       end
+      interpolate_ripper_ignored_tokens(original_code, tokens)
     end
   ensure
     $VERBOSE = verbose
   end
 
-  def find_prev_spaces(line_index)
-    return 0 if @tokens.size == 0
-    md = @tokens[0].tok.match(/(\A +)/)
-    prev_spaces = md.nil? ? 0 : md[1].count(' ')
-    line_count = 0
-    @tokens.each_with_index do |t, i|
-      if t.tok.include?("\n")
-        line_count += t.tok.count("\n")
-        if line_count >= line_index
-          return prev_spaces
-        end
-        next if t.event == :on_tstring_content || t.event == :on_words_sep
-        if (@tokens.size - 1) > i
-          md = @tokens[i + 1].tok.match(/(\A +)/)
-          prev_spaces = md.nil? ? 0 : md[1].count(' ')
-        end
-      end
-    end
-    prev_spaces
+  def prompt(opens, continue, line_num_offset)
+    ltype = ltype_from_open_tokens(opens)
+    indent_level = calc_indent_level(opens)
+    @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset)
   end
 
-  def set_auto_indent
-    if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
-      @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
-        if is_newline
-          @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: @context)
-          prev_spaces = find_prev_spaces(line_index)
-          depth_difference = check_newline_depth_difference
-          depth_difference = 0 if depth_difference < 0
-          prev_spaces + depth_difference * 2
-        else
-          code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
-          last_line = lines[line_index]&.byteslice(0, byte_pointer)
-          code += last_line if last_line
-          @tokens = self.class.ripper_lex_without_warning(code, context: @context)
-          check_corresponding_token_depth(lines, line_index)
-        end
-      end
-    end
+  def check_code_state(code)
+    check_target_code = code.gsub(/\s*\z/, '').concat("\n")
+    tokens = self.class.ripper_lex_without_warning(check_target_code, context: @context)
+    opens = IRB::NestingParser.open_tokens(tokens)
+    [tokens, opens, code_terminated?(code, tokens, opens)]
   end
 
-  def check_state(code, tokens)
-    ltype = process_literal_type(tokens)
-    indent = process_nesting_level(tokens)
-    continue = process_continue(tokens)
-    lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables)
-    code = "#{lvars_code}\n#{code}" if lvars_code
-    code_block_open = check_code_block(code, tokens)
-    [ltype, indent, continue, code_block_open]
-  end
-
-  def prompt
-    if @prompt
-      @prompt.call(@ltype, @indent, @continue, @line_no)
+  def code_terminated?(code, tokens, opens)
+    case check_code_syntax(code)
+    when :unrecoverable_error
+      true
+    when :recoverable_error
+      false
+    when :other_error
+      opens.empty? && !should_continue?(tokens)
+    when :valid
+      !should_continue?(tokens)
     end
   end
 
-  def initialize_input
-    @ltype = nil
-    @indent = 0
-    @continue = false
-    @line = ""
-    @exp_line_no = @line_no
-    @code_block_open = false
+  def save_prompt_to_context_io(opens, continue, line_num_offset)
+    # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`.
+    prompt(opens, continue, line_num_offset)
   end
 
-  def each_top_level_statement
-    initialize_input
-    catch(:TERM_INPUT) do
-      loop do
-        begin
-          prompt
-          unless l = lex
-            throw :TERM_INPUT if @line == ''
-          else
-            @line_no += l.count("\n")
-            if l == "\n"
-              @exp_line_no += 1
-              next
-            end
-            @line.concat l
-            if @code_block_open or @ltype or @continue or @indent > 0
-              next
-            end
-          end
-          if @line != "\n"
-            @line.force_encoding(@io.encoding)
-            yield @line, @exp_line_no
-          end
-          raise TerminateLineInput if @io.eof?
-          @line = ''
-          @exp_line_no = @line_no
-
-          @indent = 0
-        rescue TerminateLineInput
-          initialize_input
-          prompt
-        end
+  def readmultiline
+    save_prompt_to_context_io([], false, 0)
+
+    # multiline
+    return @input.call if @io.respond_to?(:check_termination)
+
+    # nomultiline
+    code = ''
+    line_offset = 0
+    loop do
+      line = @input.call
+      unless line
+        return code.empty? ? nil : code
       end
+
+      code << line
+      # Accept any single-line input for symbol aliases or commands that transform args
+      return code if single_line_command?(code)
+
+      tokens, opens, terminated = check_code_state(code)
+      return code if terminated
+
+      line_offset += 1
+      continue = should_continue?(tokens)
+      save_prompt_to_context_io(opens, continue, line_offset)
     end
   end
 
-  def lex
-    line = @input.call
-    if @io.respond_to?(:check_termination)
-      return line # multiline
+  def each_top_level_statement
+    loop do
+      code = readmultiline
+      break unless code
+
+      if code != "\n"
+        code.force_encoding(@io.encoding)
+        yield code, @line_no
+      end
+      @line_no += code.count("\n")
+    rescue TerminateLineInput
     end
-    code = @line + (line.nil? ? '' : line)
-    code.gsub!(/\s*\z/, '').concat("\n")
-    @tokens = self.class.ripper_lex_without_warning(code, context: @context)
-    @ltype, @indent, @continue, @code_block_open = check_state(code, @tokens)
-    line
   end
 
-  def process_continue(tokens)
-    # last token is always newline
-    if tokens.size >= 2 and tokens[-2].event == :on_regexp_end
-      # end of regexp literal
-      return false
-    elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon
-      return false
-    elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok)
-      return false
-    elsif !tokens.empty? and tokens.last.tok == "\\\n"
-      return true
-    elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n"
-      return false
-    elsif tokens.size >= 2 and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/
-      # end of literal except for regexp
-      # endless range at end of line is not a continue
-      return true
+  def should_continue?(tokens)
+    # Look at the last token and check if IRB need to continue reading next line.
+    # Example code that should continue: `a\` `a +` `a.`
+    # Trailing spaces, newline, comments are skipped
+    return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n"
+
+    tokens.reverse_each do |token|
+      case token.event
+      when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end
+        # Skip
+      when :on_regexp_end, :on_heredoc_end, :on_semicolon
+        # State is EXPR_BEG but should not continue
+        return false
+      else
+        # Endless range should not continue
+        return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/)
+
+        # EXPR_DOT and most of the EXPR_BEG should continue
+        return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT)
+      end
     end
     false
   end
 
-  def check_code_block(code, tokens)
-    return true if tokens.empty?
-    if tokens.last.event == :on_heredoc_beg
-      return true
-    end
+  def check_code_syntax(code)
+    lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables)
+    code = "#{lvars_code}\n#{code}"
 
     begin # check if parser error are available
       verbose, $VERBOSE = $VERBOSE, nil
@@ -321,6 +300,7 @@ class RubyLex
       end
     rescue EncodingError
       # This is for a hash with invalid encoding symbol, {"\xAE": 1}
+      :unrecoverable_error
     rescue SyntaxError => e
       case e.message
       when /unterminated (?:string|regexp) meets end of file/
@@ -333,7 +313,7 @@ class RubyLex
         #
         #   example:
         #     '
-        return true
+        return :recoverable_error
       when /syntax error, unexpected end-of-input/
         # "syntax error, unexpected end-of-input, expecting keyword_end"
         #
@@ -343,7 +323,7 @@ class RubyLex
         #       if false
         #         fuga
         #       end
-        return true
+        return :recoverable_error
       when /syntax error, unexpected keyword_end/
         # "syntax error, unexpected keyword_end"
         #
@@ -353,402 +333,160 @@ class RubyLex
         #
         #   example:
         #     end
-        return false
+        return :unrecoverable_error
       when /syntax error, unexpected '\.'/
         # "syntax error, unexpected '.'"
         #
         #   example:
         #     .
-        return false
+        return :unrecoverable_error
       when /unexpected tREGEXP_BEG/
         # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
         #
         #   example:
         #     method / f /
-        return false
+        return :unrecoverable_error
+      else
+        return :other_error
       end
     ensure
       $VERBOSE = verbose
     end
-
-    last_lex_state = tokens.last.state
-
-    if last_lex_state.allbits?(Ripper::EXPR_BEG)
-      return false
-    elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
-      return true
-    elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
-      return true
-    elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
-      return true
-    elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
-      return true
-    elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
-      return false
-    end
-
-    false
+    :valid
   end
 
-  def process_nesting_level(tokens)
-    indent = 0
-    in_oneliner_def = nil
-    tokens.each_with_index { |t, index|
-      # detecting one-liner method definition
-      if in_oneliner_def.nil?
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          in_oneliner_def = :ENDFN
-        end
-      else
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          # continuing
-        elsif t.state.allbits?(Ripper::EXPR_BEG)
-          if t.tok == '='
-            in_oneliner_def = :BODY
-          end
-        else
-          if in_oneliner_def == :BODY
-            # one-liner method definition
-            indent -= 1
-          end
-          in_oneliner_def = nil
-        end
-      end
-
+  def calc_indent_level(opens)
+    indent_level = 0
+    opens.each_with_index do |t, index|
       case t.event
-      when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
-        indent += 1
-      when :on_rbracket, :on_rbrace, :on_rparen
-        indent -= 1
-      when :on_kw
-        next if index > 0 and tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
-        case t.tok
-        when 'do'
-          syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index)
-          indent += 1 if syntax_of_do == :method_calling
-        when 'def', 'case', 'for', 'begin', 'class', 'module'
-          indent += 1
-        when 'if', 'unless', 'while', 'until'
-          # postfix if/unless/while/until must be Ripper::EXPR_LABEL
-          indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL)
-        when 'end'
-          indent -= 1
+      when :on_heredoc_beg
+        if opens[index + 1]&.event != :on_heredoc_beg
+          if t.tok.match?(/^<<[~-]/)
+            indent_level += 1
+          else
+            indent_level = 0
+          end
         end
+      when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick
+        # can be indented if t.tok starts with `%`
+      when :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_embexpr_beg
+        # can be indented but not indented in current implementation
+      when :on_embdoc_beg
+        indent_level = 0
+      else
+        indent_level += 1
       end
-      # percent literals are not indented
-    }
-    indent
+    end
+    indent_level
   end
 
-  def is_method_calling?(tokens, index)
-    tk = tokens[index]
-    if tk.state.anybits?(Ripper::EXPR_CMDARG) and tk.event == :on_ident
-      # The target method call to pass the block with "do".
-      return true
-    elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident
-      non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp }
-      if non_sp_index
-        prev_tk = tokens[non_sp_index]
-        if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period
-          # The target method call with receiver to pass the block with "do".
-          return true
-        end
-      end
-    end
-    false
+  FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg]
+
+  def free_indent_token?(token)
+    FREE_INDENT_TOKENS.include?(token&.event)
   end
 
-  def take_corresponding_syntax_to_kw_do(tokens, index)
-    syntax_of_do = nil
-    # Finding a syntax corresponding to "do".
-    index.downto(0) do |i|
-      tk = tokens[i]
-      # In "continue", the token isn't the corresponding syntax to "do".
-      non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
-      first_in_fomula = false
-      if non_sp_index.nil?
-        first_in_fomula = true
-      elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
-        first_in_fomula = true
-      end
-      if is_method_calling?(tokens, i)
-        syntax_of_do = :method_calling
-        break if first_in_fomula
-      elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok)
-        # A loop syntax in front of "do" found.
-        #
-        #   while cond do # also "until" or "for"
-        #   end
-        #
-        # This "do" doesn't increment indent because the loop syntax already
-        # incremented.
-        syntax_of_do = :loop_syntax
-        break if first_in_fomula
+  # Calculates the difference of pasted code's indent and indent calculated from tokens
+  def indent_difference(lines, line_results, line_index)
+    loop do
+      _tokens, prev_opens, _next_opens, min_depth = line_results[line_index]
+      open_token = prev_opens.last
+      if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token))
+        # If the leading whitespace is an indent, return the difference
+        indent_level = calc_indent_level(prev_opens.take(min_depth))
+        calculated_indent = 2 * indent_level
+        actual_indent = lines[line_index][/^ */].size
+        return actual_indent - calculated_indent
+      elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/)
+        return 0
       end
+      # If the leading whitespace is not an indent but part of a multiline token
+      # Calculate base_indent of the multiline token's beginning line
+      line_index = open_token.pos[0] - 1
     end
-    syntax_of_do
   end
 
-  def is_the_in_correspond_to_a_for(tokens, index)
-    syntax_of_in = nil
-    # Finding a syntax corresponding to "do".
-    index.downto(0) do |i|
-      tk = tokens[i]
-      # In "continue", the token isn't the corresponding syntax to "do".
-      non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
-      first_in_fomula = false
-      if non_sp_index.nil?
-        first_in_fomula = true
-      elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
-        first_in_fomula = true
-      end
-      if tk.event == :on_kw && tk.tok == 'for'
-        # A loop syntax in front of "do" found.
-        #
-        #   while cond do # also "until" or "for"
-        #   end
-        #
-        # This "do" doesn't increment indent because the loop syntax already
-        # incremented.
-        syntax_of_in = :for
-      end
-      break if first_in_fomula
+  def process_indent_level(tokens, lines, line_index, is_newline)
+    line_results = IRB::NestingParser.parse_by_line(tokens)
+    result = line_results[line_index]
+    if result
+      _tokens, prev_opens, next_opens, min_depth = result
+    else
+      # When last line is empty
+      prev_opens = next_opens = line_results.last[2]
+      min_depth = next_opens.size
     end
-    syntax_of_in
-  end
 
-  def check_newline_depth_difference
-    depth_difference = 0
-    open_brace_on_line = 0
-    in_oneliner_def = nil
-    @tokens.each_with_index do |t, index|
-      # detecting one-liner method definition
-      if in_oneliner_def.nil?
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          in_oneliner_def = :ENDFN
-        end
-      else
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          # continuing
-        elsif t.state.allbits?(Ripper::EXPR_BEG)
-          if t.tok == '='
-            in_oneliner_def = :BODY
-          end
-        else
-          if in_oneliner_def == :BODY
-            # one-liner method definition
-            depth_difference -= 1
-          end
-          in_oneliner_def = nil
-        end
-      end
+    # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation.
+    # Shortest open tokens can be calculated by `opens.take(min_depth)`
+    indent = 2 * calc_indent_level(prev_opens.take(min_depth))
 
-      case t.event
-      when :on_ignored_nl, :on_nl, :on_comment
-        if index != (@tokens.size - 1) and in_oneliner_def != :BODY
-          depth_difference = 0
-          open_brace_on_line = 0
-        end
-        next
-      when :on_sp
-        next
-      end
+    preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size
 
-      case t.event
-      when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
-        depth_difference += 1
-        open_brace_on_line += 1
-      when :on_rbracket, :on_rbrace, :on_rparen
-        depth_difference -= 1 if open_brace_on_line > 0
-      when :on_kw
-        next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
-        case t.tok
-        when 'do'
-          syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
-          depth_difference += 1 if syntax_of_do == :method_calling
-        when 'def', 'case', 'for', 'begin', 'class', 'module'
-          depth_difference += 1
-        when 'if', 'unless', 'while', 'until', 'rescue'
-          # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
-          unless t.state.allbits?(Ripper::EXPR_LABEL)
-            depth_difference += 1
-          end
-        when 'else', 'elsif', 'ensure', 'when'
-          depth_difference += 1
-        when 'in'
-          unless is_the_in_correspond_to_a_for(@tokens, index)
-            depth_difference += 1
-          end
-        when 'end'
-          depth_difference -= 1
-        end
-      end
-    end
-    depth_difference
-  end
+    prev_open_token = prev_opens.last
+    next_open_token = next_opens.last
 
-  def check_corresponding_token_depth(lines, line_index)
-    corresponding_token_depth = nil
-    is_first_spaces_of_line = true
-    is_first_printable_of_line = true
-    spaces_of_nest = []
-    spaces_at_line_head = 0
-    open_brace_on_line = 0
-    in_oneliner_def = nil
-
-    if heredoc_scope?
-      return lines[line_index][/^ */].length
+    # Calculates base indent for pasted code on the line where prev_open_token is located
+    # irb(main):001:1*   if a # base_indent is 2, indent calculated from tokens is 0
+    # irb(main):002:1*         if b # base_indent is 6, indent calculated from tokens is 2
+    # irb(main):003:0>           c # base_indent is 6, indent calculated from tokens is 4
+    if prev_open_token
+      base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max
+    else
+      base_indent = 0
     end
 
-    @tokens.each_with_index do |t, index|
-      # detecting one-liner method definition
-      if in_oneliner_def.nil?
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          in_oneliner_def = :ENDFN
-        end
+    if free_indent_token?(prev_open_token)
+      if is_newline && prev_open_token.pos[0] == line_index
+        # First newline inside free-indent token
+        base_indent + indent
       else
-        if t.state.allbits?(Ripper::EXPR_ENDFN)
-          # continuing
-        elsif t.state.allbits?(Ripper::EXPR_BEG)
-          if t.tok == '='
-            in_oneliner_def = :BODY
-          end
-        else
-          if in_oneliner_def == :BODY
-            # one-liner method definition
-            if is_first_printable_of_line
-              corresponding_token_depth = spaces_of_nest.pop
-            else
-              spaces_of_nest.pop
-              corresponding_token_depth = nil
-            end
-          end
-          in_oneliner_def = nil
-        end
+        # Accept any number of indent inside free-indent token
+        preserve_indent
       end
-
-      case t.event
-      when :on_ignored_nl, :on_nl, :on_comment, :on_heredoc_end, :on_embdoc_end
-        if in_oneliner_def != :BODY
-          corresponding_token_depth = nil
-          spaces_at_line_head = 0
-          is_first_spaces_of_line = true
-          is_first_printable_of_line = true
-          open_brace_on_line = 0
-        end
-        next
-      when :on_sp
-        spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line
-        is_first_spaces_of_line = false
-        next
+    elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg
+      if prev_open_token&.event == next_open_token&.event
+        # Accept any number of indent inside embdoc content
+        preserve_indent
+      else
+        # =begin or =end
+        0
       end
-
-      case t.event
-      when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
-        spaces_of_nest.push(spaces_at_line_head + open_brace_on_line * 2)
-        open_brace_on_line += 1
-      when :on_rbracket, :on_rbrace, :on_rparen
-        if is_first_printable_of_line
-          corresponding_token_depth = spaces_of_nest.pop
+    elsif prev_open_token&.event == :on_heredoc_beg
+      tok = prev_open_token.tok
+      if prev_opens.size <= next_opens.size
+        if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token
+          # First line in heredoc
+          tok.match?(/^<<[-~]/) ? base_indent + indent : indent
+        elsif tok.match?(/^<<~/)
+          # Accept extra indent spaces inside `<<~` heredoc
+          [base_indent + indent, preserve_indent].max
         else
-          spaces_of_nest.pop
-          corresponding_token_depth = nil
-        end
-        open_brace_on_line -= 1
-      when :on_kw
-        next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
-        case t.tok
-        when 'do'
-          syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
-          if syntax_of_do == :method_calling
-            spaces_of_nest.push(spaces_at_line_head)
-          end
-        when 'def', 'case', 'for', 'begin', 'class', 'module'
-          spaces_of_nest.push(spaces_at_line_head)
-        when 'rescue'
-          unless t.state.allbits?(Ripper::EXPR_LABEL)
-            corresponding_token_depth = spaces_of_nest.last
-          end
-        when 'if', 'unless', 'while', 'until'
-          # postfix if/unless/while/until must be Ripper::EXPR_LABEL
-          unless t.state.allbits?(Ripper::EXPR_LABEL)
-            spaces_of_nest.push(spaces_at_line_head)
-          end
-        when 'else', 'elsif', 'ensure', 'when'
-          corresponding_token_depth = spaces_of_nest.last
-        when 'in'
-          if in_keyword_case_scope?
-            corresponding_token_depth = spaces_of_nest.last
-          end
-        when 'end'
-          if is_first_printable_of_line
-            corresponding_token_depth = spaces_of_nest.pop
-          else
-            spaces_of_nest.pop
-            corresponding_token_depth = nil
-          end
+          # Accept any number of indent inside other heredoc
+          preserve_indent
         end
+      else
+        # Heredoc close
+        prev_line_indent_level = calc_indent_level(prev_opens)
+        tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0
       end
-      is_first_spaces_of_line = false
-      is_first_printable_of_line = false
+    else
+      base_indent + indent
     end
-    corresponding_token_depth
   end
 
-  def check_string_literal(tokens)
-    i = 0
-    start_token = []
-    end_type = []
-    pending_heredocs = []
-    while i < tokens.size
-      t = tokens[i]
-      case t.event
-      when *end_type.last
-        start_token.pop
-        end_type.pop
-      when :on_tstring_beg
-        start_token << t
-        end_type << [:on_tstring_end, :on_label_end]
-      when :on_regexp_beg
-        start_token << t
-        end_type << :on_regexp_end
-      when :on_symbeg
-        acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick}
-        if (i + 1) < tokens.size
-          if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st }
-            start_token << t
-            end_type << :on_tstring_end
-          else
-            i += 1
-          end
-        end
-      when :on_backtick
-        if t.state.allbits?(Ripper::EXPR_BEG)
-          start_token << t
-          end_type << :on_tstring_end
-        end
-      when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
-        start_token << t
-        end_type << :on_tstring_end
-      when :on_heredoc_beg
-        pending_heredocs << t
-      end
+  LTYPE_TOKENS = %i[
+    on_heredoc_beg on_tstring_beg
+    on_regexp_beg on_symbeg on_backtick
+    on_symbols_beg on_qsymbols_beg
+    on_words_beg on_qwords_beg
+  ]
 
-      if pending_heredocs.any? && t.tok.include?("\n")
-        pending_heredocs.reverse_each do |t|
-          start_token << t
-          end_type << :on_heredoc_end
-        end
-        pending_heredocs = []
-      end
-      i += 1
+  def ltype_from_open_tokens(opens)
+    start_token = opens.reverse_each.find do |tok|
+      LTYPE_TOKENS.include?(tok.event)
     end
-    pending_heredocs.first || start_token.last
-  end
-
-  def process_literal_type(tokens)
-    start_token = check_string_literal(tokens)
-    return nil if start_token == ""
+    return nil unless start_token
 
     case start_token&.event
     when :on_tstring_beg
@@ -801,47 +539,16 @@ class RubyLex
         end
       end
 
-      if first_token.nil?
-        return false
-      elsif first_token && first_token.state == Ripper::EXPR_DOT
-        return false
-      else
+      if first_token && first_token.state != Ripper::EXPR_DOT
         tokens_without_last_line = tokens[0..index]
-        ltype = process_literal_type(tokens_without_last_line)
-        indent = process_nesting_level(tokens_without_last_line)
-        continue = process_continue(tokens_without_last_line)
-        code_block_open = check_code_block(tokens_without_last_line.map(&:tok).join(''), tokens_without_last_line)
-        if ltype or indent > 0 or continue or code_block_open
-          return false
-        else
-          return last_line_tokens.map(&:tok).join('')
+        code_without_last_line = tokens_without_last_line.map(&:tok).join
+        opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line)
+        if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
+          return last_line_tokens.map(&:tok).join
         end
       end
     end
     false
   end
-
-  private
-
-  def heredoc_scope?
-    heredoc_tokens = @tokens.select { |t| [:on_heredoc_beg, :on_heredoc_end].include?(t.event) }
-    heredoc_tokens[-1]&.event == :on_heredoc_beg
-  end
-
-  def in_keyword_case_scope?
-    kw_tokens = @tokens.select { |t| t.event == :on_kw && ['case', 'for', 'end'].include?(t.tok) }
-    counter = 0
-    kw_tokens.reverse.each do |t|
-      if t.tok == 'case'
-        return true if counter.zero?
-        counter += 1
-      elsif t.tok == 'for'
-        counter += 1
-      elsif t.tok == 'end'
-        counter -= 1
-      end
-    end
-    false
-  end
 end
 # :startdoc: