katakata_irb 0.1.0

data/lib/katakata_irb/reline_patches/indent.patch

@@ -0,0 +1,25 @@
+diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb
+index bbf5e6c..c9e613e 100644
+--- a/lib/reline/line_editor.rb
++++ b/lib/reline/line_editor.rb
+@@ -1707,17 +1707,18 @@ class Reline::LineEditor
+     end
+     new_lines = whole_lines
+     new_indent = @auto_indent_proc.(new_lines, @line_index, @byte_pointer, @check_new_auto_indent)
+-    new_indent = @cursor_max if new_indent&.> @cursor_max
+     if new_indent&.>= 0
+       md = new_lines[@line_index].match(/\A */)
+       prev_indent = md[0].count(' ')
+       if @check_new_auto_indent
+-        @buffer_of_lines[@line_index] = ' ' * new_indent + @buffer_of_lines[@line_index].lstrip
++        line = @buffer_of_lines[@line_index] = ' ' * new_indent + @buffer_of_lines[@line_index].lstrip
+         @cursor = new_indent
++        @cursor_max = calculate_width(line)
+         @byte_pointer = new_indent
+       else
+         @line = ' ' * new_indent + @line.lstrip
+         @cursor += new_indent - prev_indent
++        @cursor_max = calculate_width(@line)
+         @byte_pointer += new_indent - prev_indent
+       end
+     end

data/lib/katakata_irb/reline_patches/raw.patch

@@ -0,0 +1,95 @@
+diff --git a/lib/reline.rb b/lib/reline.rb
+index f22b573..8716a0c 100644
+--- a/lib/reline.rb
++++ b/lib/reline.rb
+@@ -281,19 +281,21 @@ module Reline
+     Reline::DEFAULT_DIALOG_CONTEXT = Array.new
+ 
+     def readmultiline(prompt = '', add_hist = false, &confirm_multiline_termination)
+-      unless confirm_multiline_termination
+-        raise ArgumentError.new('#readmultiline needs block to confirm multiline termination')
+-      end
+-      inner_readline(prompt, add_hist, true, &confirm_multiline_termination)
++      Reline::IOGate.with_raw_input do
++        unless confirm_multiline_termination
++          raise ArgumentError.new('#readmultiline needs block to confirm multiline termination')
++        end
++        inner_readline(prompt, add_hist, true, &confirm_multiline_termination)
+ 
+-      whole_buffer = line_editor.whole_buffer.dup
+-      whole_buffer.taint if RUBY_VERSION < '2.7'
+-      if add_hist and whole_buffer and whole_buffer.chomp("\n").size > 0
+-        Reline::HISTORY << whole_buffer
+-      end
++        whole_buffer = line_editor.whole_buffer.dup
++        whole_buffer.taint if RUBY_VERSION < '2.7'
++        if add_hist and whole_buffer and whole_buffer.chomp("\n").size > 0
++          Reline::HISTORY << whole_buffer
++        end
+ 
+-      line_editor.reset_line if line_editor.whole_buffer.nil?
+-      whole_buffer
++        line_editor.reset_line if line_editor.whole_buffer.nil?
++        whole_buffer
++      end
+     end
+ 
+     def readline(prompt = '', add_hist = false)
+diff --git a/lib/reline/ansi.rb b/lib/reline/ansi.rb
+index ab147a6..ccebe15 100644
+--- a/lib/reline/ansi.rb
++++ b/lib/reline/ansi.rb
+@@ -142,6 +142,10 @@ class Reline::ANSI
+     @@output = val
+   end
+ 
++  def self.with_raw_input
++    @@input.raw { yield }
++  end
++
+   @@buf = []
+   def self.inner_getc
+     unless @@buf.empty?
+diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb
+index 92c76cb..9929846 100644
+--- a/lib/reline/general_io.rb
++++ b/lib/reline/general_io.rb
+@@ -31,6 +31,10 @@ class Reline::GeneralIO
+     @@input = val
+   end
+ 
++  def self.with_raw_input
++    yield
++  end
++
+   def self.getc
+     unless @@buf.empty?
+       return @@buf.shift
+diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb
+index 6acf969..e3985a3 100644
+--- a/lib/reline/line_editor.rb
++++ b/lib/reline/line_editor.rb
+@@ -452,7 +452,7 @@ class Reline::LineEditor
+       new_lines = whole_lines
+       prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines)
+       modify_lines(new_lines).each_with_index do |line, index|
+-        @output.write "#{prompt_list ? prompt_list[index] : prompt}#{line}\n"
++        @output.write "#{prompt_list ? prompt_list[index] : prompt}#{line}\r\n"
+         Reline::IOGate.erase_after_cursor
+       end
+       @output.flush
+diff --git a/lib/reline/windows.rb b/lib/reline/windows.rb
+index b952329..7ea2a00 100644
+--- a/lib/reline/windows.rb
++++ b/lib/reline/windows.rb
+@@ -291,6 +291,10 @@ class Reline::Windows
+     end
+   end
+ 
++  def self.with_raw_input
++    yield
++  end
++
+   def self.getc
+     check_input_event
+     @@output_buf.shift

data/lib/katakata_irb/reline_patches/scrollbar.patch

@@ -0,0 +1,43 @@
+diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb
+index e3985a3..8d785db 100644
+--- a/lib/reline/line_editor.rb
++++ b/lib/reline/line_editor.rb
+@@ -680,6 +680,9 @@ class Reline::LineEditor
+     end
+     height = dialog_render_info.height || DIALOG_DEFAULT_HEIGHT
+     height = dialog.contents.size if dialog.contents.size < height
++    if dialog.scroll_top >= dialog.contents.size - height
++      dialog.scroll_top = dialog.contents.size - height
++    end
+     if dialog.contents.size > height
+       if dialog.pointer
+         if dialog.pointer < 0
+@@ -690,17 +693,20 @@ class Reline::LineEditor
+           dialog.scroll_top = dialog.pointer
+         end
+         pointer = dialog.pointer - dialog.scroll_top
++      else
++        dialog.scroll_top = 0
+       end
+       dialog.contents = dialog.contents[dialog.scroll_top, height]
+     end
+-    if dialog.contents and dialog.scroll_top >= dialog.contents.size
+-      dialog.scroll_top = dialog.contents.size - height
++    if dialog_render_info.contents and dialog.scroll_top > dialog_render_info.contents.size - height
++      dialog.scroll_top = dialog_render_info.contents.size - height
+     end
+     if dialog_render_info.scrollbar and dialog_render_info.contents.size > height
+       bar_max_height = height * 2
+       moving_distance = (dialog_render_info.contents.size - height) * 2
+       position_ratio = dialog.scroll_top.zero? ? 0.0 : ((dialog.scroll_top * 2).to_f / moving_distance)
+       bar_height = (bar_max_height * ((dialog.contents.size * 2).to_f / (dialog_render_info.contents.size * 2))).floor.to_i
++      bar_height = 1 if bar_height.zero?
+       dialog.scrollbar_pos = ((bar_max_height - bar_height) * position_ratio).floor.to_i
+     else
+       dialog.scrollbar_pos = nil
+@@ -745,4 +751,4 @@ class Reline::LineEditor
+-      if dialog.scrollbar_pos and (dialog.scrollbar_pos != old_dialog.scrollbar_pos or dialog.column != old_dialog.column)
++      if dialog.scrollbar_pos
+         @output.write "\e[37m"
+         if dialog.scrollbar_pos <= (i * 2) and (i * 2 + 1) < (dialog.scrollbar_pos + bar_height)
+           @output.write @full_block

data/lib/katakata_irb/reline_patches/wholelines.patch

@@ -0,0 +1,102 @@
+diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb
+index 8153aab..1c33a4b 100644
+--- a/lib/reline/line_editor.rb
++++ b/lib/reline/line_editor.rb
+@@ -449,12 +449,8 @@ class Reline::LineEditor
+       Reline::IOGate.move_cursor_up(@first_line_started_from + @started_from - @scroll_partial_screen)
+       Reline::IOGate.move_cursor_column(0)
+       @scroll_partial_screen = nil
+-      prompt, prompt_width, prompt_list = check_multiline_prompt(whole_lines)
+-      if @previous_line_index
+-        new_lines = whole_lines(index: @previous_line_index, line: @line)
+-      else
+-        new_lines = whole_lines
+-      end
++      new_lines = whole_lines
++      prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines)
+       modify_lines(new_lines).each_with_index do |line, index|
+         @output.write "#{prompt_list ? prompt_list[index] : prompt}#{line}\n"
+         Reline::IOGate.erase_after_cursor
+@@ -490,11 +486,7 @@ class Reline::LineEditor
+     if @is_multiline
+       if finished?
+         # Always rerender on finish because output_modifier_proc may return a different output.
+-        if @previous_line_index
+-          new_lines = whole_lines(index: @previous_line_index, line: @line)
+-        else
+-          new_lines = whole_lines
+-        end
++        new_lines = whole_lines
+         line = modify_lines(new_lines)[@line_index]
+         clear_dialog
+         prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines)
+@@ -1013,11 +1005,7 @@ class Reline::LineEditor
+   end
+ 
+   private def rerender_changed_current_line
+-    if @previous_line_index
+-      new_lines = whole_lines(index: @previous_line_index, line: @line)
+-    else
+-      new_lines = whole_lines
+-    end
++    new_lines = whole_lines
+     prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines)
+     all_height = calculate_height_by_lines(new_lines, prompt_list || prompt)
+     diff = all_height - @highest_in_all
+@@ -1698,7 +1686,7 @@ class Reline::LineEditor
+     return if not @check_new_auto_indent and @previous_line_index # move cursor up or down
+     if @check_new_auto_indent and @previous_line_index and @previous_line_index > 0 and @line_index > @previous_line_index
+       # Fix indent of a line when a newline is inserted to the next
+-      new_lines = whole_lines(index: @previous_line_index, line: @line)
++      new_lines = whole_lines
+       new_indent = @auto_indent_proc.(new_lines[0..-3].push(''), @line_index - 1, 0, true)
+       md = @line.match(/\A */)
+       prev_indent = md[0].count(' ')
+@@ -1713,11 +1701,7 @@ class Reline::LineEditor
+         @line = ' ' * new_indent + @line.lstrip
+       end
+     end
+-    if @previous_line_index
+-      new_lines = whole_lines(index: @previous_line_index, line: @line)
+-    else
+-      new_lines = whole_lines
+-    end
++    new_lines = whole_lines
+     new_indent = @auto_indent_proc.(new_lines, @line_index, @byte_pointer, @check_new_auto_indent)
+     new_indent = @cursor_max if new_indent&.> @cursor_max
+     if new_indent&.>= 0
+@@ -1803,11 +1787,7 @@ class Reline::LineEditor
+       target = before
+     end
+     if @is_multiline
+-      if @previous_line_index
+-        lines = whole_lines(index: @previous_line_index, line: @line)
+-      else
+-        lines = whole_lines
+-      end
++      lines = whole_lines
+       if @line_index > 0
+         preposing = lines[0..(@line_index - 1)].join("\n") + "\n" + preposing
+       end
+@@ -1907,7 +1887,7 @@ class Reline::LineEditor
+     @cursor_max = calculate_width(@line)
+   end
+ 
+-  def whole_lines(index: @line_index, line: @line)
++  def whole_lines(index: @previous_line_index || @line_index, line: @line)
+     temp_lines = @buffer_of_lines.dup
+     temp_lines[index] = line
+     temp_lines
+@@ -1917,11 +1897,7 @@ class Reline::LineEditor
+     if @buffer_of_lines.size == 1 and @line.nil?
+       nil
+     else
+-      if @previous_line_index
+-        whole_lines(index: @previous_line_index, line: @line).join("\n")
+-      else
+-        whole_lines.join("\n")
+-      end
++      whole_lines.join("\n")
+     end
+   end
+ 

data/lib/katakata_irb/ruby_lex_patch.rb

@@ -0,0 +1,197 @@
+require 'irb'
+require_relative 'trex'
+
+module KatakataIrb::RubyLexPatch
+  def self.patch_to_ruby_lex
+    (RubyLex.instance_methods(false) - [:initialize_input, :set_prompt, :process_continue, :check_code_block]).each { RubyLex.remove_method _1 }
+    RubyLex.prepend self
+  end
+
+  def self.complete_tokens(code, context: nil)
+    incomplete_tokens = RubyLex.ripper_lex_without_warning(code, context: context)
+    KatakataIrb::TRex.interpolate_ripper_ignored_tokens(code, incomplete_tokens)
+  end
+
+  def calc_nesting_depth(tokens)
+    indent_level = 0
+    nesting_level = 0
+    tokens.each_with_index do |t, index|
+      case t.event
+      when :on_heredoc_beg
+        if tokens[index + 1]&.event != :on_heredoc_beg
+          if t.tok.start_with?('<<~')
+            indent_level += 1
+          else
+            indent_level = 0
+          end
+        end
+      when :on_tstring_beg, :on_regexp_beg
+        indent_level += 1 if t.tok[0] == '%'
+      when :on_embdoc_beg
+        indent_level = 0
+      else
+        nesting_level += 1
+        indent_level += 1
+      end
+    end
+    [indent_level, nesting_level]
+  end
+
+  def process_indent_level(tokens)
+    opens, heredocs = KatakataIrb::TRex.parse(tokens)
+    indent, _nesting = calc_nesting_depth(opens.map(&:first) + heredocs)
+    indent * 2
+  end
+
+  def check_corresponding_token_depth(tokens, line_index)
+    lines, = KatakataIrb::TRex.parse_line(tokens)
+    result = lines[line_index]
+    return unless result
+    _tokens, prev, opens, min_depth = result
+    depth, = calc_nesting_depth(opens.take(min_depth).map(&:first))
+    prev_depth, = calc_nesting_depth(prev.map(&:first))
+    depth * 2 if depth < prev_depth
+  end
+
+  def ltype_from_open_tokens(opens)
+    return nil if opens.empty?
+    case opens.last.tok
+    when ?`, /^<<[-~]?`/, /^%x.$/
+      ?`
+    when ?', /^<<[-~]?'/, /^%q.$/
+      ?'
+    when ?", /^<</, /^%.$/, /^%Q.$/
+      ?"
+    when ":'", ':"', ':', /^%s$/
+      ':'
+    when /^%[iwIW]$/
+      ']'
+    when '/', /^%r.$/
+      '/'
+    end
+  end
+
+  def check_termination_in_prev_line(code, context: nil)
+    *lines, last_line = code.lines
+    last_line if lines.any? && check_termination(lines.join, context: context)
+  end
+
+  def check_termination(code, context: nil)
+    tokens = KatakataIrb::RubyLexPatch.complete_tokens(code, context: context)
+    opens, heredocs = KatakataIrb::TRex.parse(tokens)
+    opens.empty? && heredocs.empty? && !process_continue(tokens)
+  end
+
+  def set_input(io, p = nil, context: nil, &block)
+    @io = io
+    if @io.respond_to?(:check_termination)
+      @io.check_termination do |code|
+        if Reline::IOGate.in_pasting?
+          lex = RubyLex.new
+          rest = lex.check_termination_in_prev_line(code, context: context)
+          if rest
+            Reline.delete_text
+            rest.bytes.reverse_each do |c|
+              Reline.ungetc(c)
+            end
+            true
+          else
+            false
+          end
+        else
+          check_termination(code, context: context)
+        end
+      end
+    end
+    if @io.respond_to?(:dynamic_prompt)
+      @io.dynamic_prompt do |lines|
+        lines << '' if lines.empty?
+        code = lines.map{ |l| l + "\n" }.join
+        tokens = KatakataIrb::RubyLexPatch.complete_tokens code, context: context
+        lines, _unclosed_heredocs = KatakataIrb::TRex.parse_line(tokens)
+        continue = false
+        lines.map.with_index do |(line, _prev_opens, next_opens), line_num_offset|
+          unless (c = process_continue(line.map(&:first))).nil?
+            continue = c
+          end
+          prompt next_opens, continue, line_num_offset
+        end
+      end
+    end
+
+    if p.respond_to?(:call)
+      @input = p
+    elsif block_given?
+      @input = block
+    else
+      @input = Proc.new{@io.gets}
+    end
+  end
+
+  def set_auto_indent(context)
+    if @io.respond_to?(:auto_indent) and context.auto_indent_mode
+      @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
+        if is_newline
+          tokens = KatakataIrb::RubyLexPatch.complete_tokens(lines[0..line_index].join("\n"), context: context)
+          process_indent_level tokens
+        else
+          code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
+          last_line = lines[line_index]&.byteslice(0, byte_pointer)
+          code += last_line if last_line
+          tokens = KatakataIrb::RubyLexPatch.complete_tokens(code, context: context)
+          check_corresponding_token_depth(tokens, line_index)
+        end
+      end
+    end
+  end
+
+  def prompt(opens, continue, line_num_offset)
+    ltype = ltype_from_open_tokens opens.map(&:first)
+    _indent, nesting_level = calc_nesting_depth opens.map(&:first)
+    @prompt.call(ltype, nesting_level, opens.any? || continue, @line_no + line_num_offset)
+  end
+
+  def store_prompt_to_irb(...)
+    prompt(...) # TODO: do not use this. change the api. example: @input.call(prompt)
+  end
+
+  def readmultiline(context)
+    if @io.respond_to? :check_termination
+      store_prompt_to_irb [], false, 0
+      @input.call
+    else
+      # nomultiline
+      line = ''
+      line_offset = 0
+      store_prompt_to_irb [], false, 0
+      loop do
+        l = @input.call
+        next if l.nil?
+        line << l
+        tokens = KatakataIrb::RubyLexPatch.complete_tokens(line, context: context)
+        _line, _prev_opens, next_opens = KatakataIrb::TRex.parse_line(tokens).first.last
+        return line if next_opens.empty?
+        line_offset += 1
+        store_prompt_to_irb next_opens, true, line_offset
+      end
+    end
+  end
+
+  def each_top_level_statement(context = nil)
+    initialize_input
+    loop do
+      begin
+        line = readmultiline(context)
+        break unless line
+        if line != "\n"
+          line.force_encoding(@io.encoding)
+          yield line, @line_no
+        end
+        @line_no += line.count("\n")
+        raise RubyLex::TerminateLineInput if @io.eof?
+      rescue RubyLex::TerminateLineInput
+        initialize_input
+      end
+    end
+  end
+end

data/lib/katakata_irb/trex.rb

@@ -0,0 +1,207 @@
+module KatakataIrb; end
+module KatakataIrb::TRex
+  def self.interpolate_ripper_ignored_tokens(code, tokens)
+    line_positions = code.lines.reduce([0]) { _1 << _1.last + _2.bytesize }
+    prev_byte_pos = 0
+    interpolated = []
+    prev_line = 1
+    event = :on_ignored_by_ripper
+    tokens.each do |t|
+      line, col = t.pos
+      byte_pos = line_positions[line - 1] + col
+      if prev_byte_pos < byte_pos
+        tok = code.byteslice(prev_byte_pos...byte_pos)
+        pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
+        interpolated << Ripper::Lexer::Elem.new(pos, event, tok, 0)
+        prev_line += tok.count("\n")
+      end
+      interpolated << t
+      prev_byte_pos = byte_pos + t.tok.bytesize
+      prev_line += t.tok.count("\n")
+    end
+    if prev_byte_pos < code.bytesize
+      tok = code.byteslice(prev_byte_pos..)
+      pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]]
+      interpolated << Ripper::Lexer::Elem.new(pos, event, tok, 0)
+    end
+    interpolated
+  end
+
+  def self.parse(tokens)
+    opens = []
+    pending_heredocs = []
+    first_token_on_line = true
+    tokens.each_with_index do |t, index|
+      skip = false
+      last_tok, state, args = opens.last
+      case state
+      when :in_unquoted_symbol
+        opens.pop
+        skip = true if %i[on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick].include? t.event
+      when :in_method_head
+        unless %i[on_sp on_ignored_nl].include?(t.event)
+          next_args = []
+          body = nil
+          if args.include? :receiver
+            case t.event
+            when :on_lparen, :on_ivar, :on_gvar, :on_cvar
+              next_args << :dot
+            when :on_kw
+              if t.tok in 'self' | 'true' | 'false' | 'nil'
+                next_args.push :arg, :dot
+              else
+                skip = true
+                next_args << :arg
+              end
+            when :on_op
+              skip = true
+              next_args << :arg
+            when :on_ident, :on_const
+              next_args.push :arg, :dot
+            end
+          end
+          if args.include? :dot
+            next_args << :name if t.event == :on_period || (t.event == :on_op && t.tok == '::')
+          end
+          if args.include? :name
+            if %i[on_ident on_const on_op on_kw].include? t.event
+              next_args << :arg
+              skip = true
+            end
+          end
+          if args.include? :arg
+            case t.event
+            when :on_op
+              body = :oneliner if t.tok == '='
+            when :on_nl, :on_semicolon
+              body = :normal
+            when :on_lparen
+              next_args << :eq
+            else
+              next_args << :arg_without_paren
+            end
+          end
+          if args.include? :eq
+            if t.event == :on_op && t.tok == '='
+              body = :oneliner
+            elsif t.event != :on_embdoc_beg
+              body = :normal
+            end
+          end
+          if args.include? :args_without_paren
+            body = :normal if %i[on_semicolon on_nl].include? t.event
+          end
+          if body == :oneliner
+            opens.pop
+          elsif body
+            opens.pop
+            opens << [last_tok, nil]
+          else
+            opens.pop
+            opens << [last_tok, :in_method_head, next_args]
+          end
+        end
+      when :in_for_while_until_condition
+        if t.event == :on_semicolon || t.event == :on_nl || (t.event == :on_kw && t.tok == 'do')
+          skip = true if t.event == :on_kw && t.tok == 'do'
+          opens.pop
+          opens << [last_tok, nil]
+        end
+      end
+
+      unless skip
+        case t.event
+        when :on_kw
+          case t.tok
+          when 'begin', 'class', 'module', 'do', 'case'
+            opens << [t, nil]
+          when 'end'
+            opens.pop
+          when 'def'
+            opens << [t, :in_method_head, [:receiver, :name]]
+          when 'if', 'unless'
+            unless t.state.allbits?(Ripper::EXPR_LABEL)
+              opens << [t, nil]
+            end
+          when 'while', 'until'
+            unless t.state.allbits?(Ripper::EXPR_LABEL)
+              opens << [t, :in_for_while_until_condition]
+            end
+          when 'ensure', 'rescue'
+            unless t.state.allbits?(Ripper::EXPR_LABEL)
+              opens.pop
+              opens << [t, nil]
+            end
+          when 'elsif', 'else', 'when'
+            opens.pop
+            opens << [t, nil]
+          when 'for'
+            opens << [t, :in_for_while_until_condition]
+          when 'in'
+            if last_tok&.event == :on_kw && %w[case in].include?(last_tok.tok) && first_token_on_line
+              opens.pop
+              opens << [t, nil]
+            end
+          end
+        when :on_lparen, :on_lbracket, :on_lbrace, :on_tlambeg, :on_embexpr_beg, :on_embdoc_beg
+          opens << [t, nil]
+        when :on_rparen, :on_rbracket, :on_rbrace, :on_embexpr_end, :on_embdoc_end
+          opens.pop
+        when :on_heredoc_beg
+          pending_heredocs << t
+        when :on_heredoc_end
+          opens.pop
+        when :on_backtick
+          opens << [t, nil] if t.state.allbits?(Ripper::EXPR_BEG)
+        when :on_tstring_beg, :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_regexp_beg
+          opens << [t, nil]
+        when :on_tstring_end, :on_regexp_end, :on_label_end
+          opens.pop
+        when :on_symbeg
+          if t.tok == ':'
+            opens << [t, :in_unquoted_symbol]
+          else
+            opens << [t, nil]
+          end
+        end
+      end
+      if t.event == :on_nl || t.event == :on_semicolon
+        first_token_on_line = true
+      elsif t.event != :on_sp
+        first_token_on_line = false
+      end
+      if pending_heredocs.any? && t.tok.include?("\n")
+        pending_heredocs.reverse_each { opens << [_1, nil] }
+        pending_heredocs = []
+      end
+      yield t, index, opens if block_given?
+    end
+    [opens, pending_heredocs.reverse]
+  end
+
+  def self.parse_line(tokens)
+    line_tokens = []
+    prev_opens = []
+    min_depth = 0
+    output = []
+    last_opens, unclosed_heredocs = KatakataIrb::TRex.parse(tokens) do |t, _index, opens|
+      depth = t == opens.last&.first ? opens.size - 1 : opens.size
+      min_depth = depth if depth < min_depth
+      if t.tok.include? "\n"
+        t.tok.each_line do |line|
+          line_tokens << [t, line]
+          next if line[-1] != "\n"
+          next_opens = opens.dup
+          output << [line_tokens, prev_opens, next_opens, min_depth]
+          prev_opens = next_opens
+          min_depth = prev_opens.size
+          line_tokens = []
+        end
+      else
+        line_tokens << [t, t.tok]
+      end
+    end
+    output << [line_tokens, prev_opens, last_opens, min_depth] if line_tokens.any?
+    [output, unclosed_heredocs]
+  end
+end