irb 1.7.0 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa3c9603f48d730988d34f454a10b58021caa677ab8832d9764e39e61bb49abb
-  data.tar.gz: c46d1e32c81a8f22593e174af095070159af6a61508c97b3ef4b0b9d28301d58
+  metadata.gz: 9400febd023df0efde88c58ff524d4261ffe727404faf5cc719036e8691b7804
+  data.tar.gz: c46f543dc01f54f1ef6eaa37c2fdad212e06ffeecec0775e65f47423b68f8f7b
 SHA512:
-  metadata.gz: 3e99185c909180d01a3310f7db11437181268b930d267f2fe365c06cd7a442ad3a8cdf7d5863fb19e17885a23aad46b993a2e5c21fb8632ff2f91285e469be4b
-  data.tar.gz: 44978be86d00d0fc435e81e4cadc5abad08d47519a189db02e34d7eb3435e9ac0cc0adf470e845edba1e7d866574298e1b7b8bba5190d8be538c396cd0179d07
+  metadata.gz: ecca8c833a94435b5d4b2abfea2669b9e6d688976d6010cff13f8440651c3379616b202e2be101bf78ac706ca108abe2b98871984e02f8a3f295522b6e820c0e
+  data.tar.gz: 7877a1a4857f5b9163745057d8a9fdf6440f40272b71699fc481c889520c40cf77f07e3f48488fec7d8ef086d89cf0463900bc8257ba72447bf3164595081f55

data/irb.gemspec

@@ -41,5 +41,5 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7")
 
-  spec.add_dependency "reline", ">= 0.3.0"
+  spec.add_dependency "reline", ">= 0.3.6"
 end

data/lib/irb/cmd/irb_info.rb

@@ -11,24 +11,20 @@ module IRB
       description "Show information about IRB."
 
       def execute
-        Class.new {
-          def inspect
-            str  = "Ruby version: #{RUBY_VERSION}\n"
-            str += "IRB version: #{IRB.version}\n"
-            str += "InputMethod: #{IRB.CurrentContext.io.inspect}\n"
-            str += ".irbrc path: #{IRB.rc_file}\n" if File.exist?(IRB.rc_file)
-            str += "RUBY_PLATFORM: #{RUBY_PLATFORM}\n"
-            str += "LANG env: #{ENV["LANG"]}\n" if ENV["LANG"] && !ENV["LANG"].empty?
-            str += "LC_ALL env: #{ENV["LC_ALL"]}\n" if ENV["LC_ALL"] && !ENV["LC_ALL"].empty?
-            str += "East Asian Ambiguous Width: #{Reline.ambiguous_width.inspect}\n"
-            if RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/
-              codepage = `chcp`.b.sub(/.*: (\d+)\n/, '\1')
-              str += "Code page: #{codepage}\n"
-            end
-            str
-          end
-          alias_method :to_s, :inspect
-        }.new
+        str  = "Ruby version: #{RUBY_VERSION}\n"
+        str += "IRB version: #{IRB.version}\n"
+        str += "InputMethod: #{IRB.CurrentContext.io.inspect}\n"
+        str += ".irbrc path: #{IRB.rc_file}\n" if File.exist?(IRB.rc_file)
+        str += "RUBY_PLATFORM: #{RUBY_PLATFORM}\n"
+        str += "LANG env: #{ENV["LANG"]}\n" if ENV["LANG"] && !ENV["LANG"].empty?
+        str += "LC_ALL env: #{ENV["LC_ALL"]}\n" if ENV["LC_ALL"] && !ENV["LC_ALL"].empty?
+        str += "East Asian Ambiguous Width: #{Reline.ambiguous_width.inspect}\n"
+        if RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+          codepage = `chcp`.b.sub(/.*: (\d+)\n/, '\1')
+          str += "Code page: #{codepage}\n"
+        end
+        puts str
+        nil
       end
     end
   end

data/lib/irb/cmd/nop.rb

@@ -30,23 +30,19 @@ module IRB
         end
       end
 
-      def self.execute(conf, *opts, **kwargs, &block)
-        command = new(conf)
+      def self.execute(irb_context, *opts, **kwargs, &block)
+        command = new(irb_context)
         command.execute(*opts, **kwargs, &block)
       rescue CommandArgumentError => e
         puts e.message
       end
 
-      def initialize(conf)
-        @irb_context = conf
+      def initialize(irb_context)
+        @irb_context = irb_context
       end
 
       attr_reader :irb_context
 
-      def irb
-        @irb_context.irb
-      end
-
       def execute(*opts)
         #nop
       end

data/lib/irb/cmd/show_cmds.rb

@@ -14,7 +14,7 @@ module IRB
       def execute(*args)
         commands_info = IRB::ExtendCommandBundle.all_commands_info
         commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] }
-        longest_cmd_name_length = commands_info.map { |c| c[:display_name] }.max { |a, b| a.length <=> b.length }.length
+        longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max
 
         output = StringIO.new
 

data/lib/irb/cmd/show_source.rb

@@ -58,9 +58,9 @@ module IRB
           tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk|
             code = lines[0..lnum].join
             prev_tokens.concat chunk
-            continue = lex.process_continue(prev_tokens)
-            code_block_open = lex.check_code_block(code, prev_tokens)
-            if !continue && !code_block_open
+            continue = lex.should_continue?(prev_tokens)
+            syntax = lex.check_code_syntax(code)
+            if !continue && syntax == :valid
               return first_line + lnum
             end
           end

data/lib/irb/completion.rb

@@ -218,7 +218,7 @@ module IRB
         else
           sym = $1
           candidates = Symbol.all_symbols.collect do |s|
-            ":" + s.id2name.encode(Encoding.default_external)
+            s.inspect
           rescue EncodingError
             # ignore
           end
@@ -233,7 +233,7 @@ module IRB
         if doc_namespace
           candidates.find { |i| i == receiver }
         else
-          candidates.grep(/^#{receiver}/).collect{|e| "::" + e}
+          candidates.grep(/^#{Regexp.quote(receiver)}/).collect{|e| "::" + e}
         end
 
       when /^([A-Z].*)::([^:.]*)$/

data/lib/irb/context.rb

@@ -8,6 +8,7 @@ require_relative "workspace"
 require_relative "inspector"
 require_relative "input-method"
 require_relative "output-method"
+require_relative "history"
 
 module IRB
   # A class that wraps the current state of the irb session, including the
@@ -151,6 +152,27 @@ module IRB
       @command_aliases = IRB.conf[:COMMAND_ALIASES]
     end
 
+    def save_history=(val)
+      IRB.conf[:SAVE_HISTORY] = val
+      if val
+        (IRB.conf[:MAIN_CONTEXT] || self).init_save_history
+      end
+    end
+
+    def save_history
+      IRB.conf[:SAVE_HISTORY]
+    end
+
+    # A copy of the default <code>IRB.conf[:HISTORY_FILE]</code>
+    def history_file
+      IRB.conf[:HISTORY_FILE]
+    end
+
+    # Set <code>IRB.conf[:HISTORY_FILE]</code> to the given +hist+.
+    def history_file=(hist)
+      IRB.conf[:HISTORY_FILE] = hist
+    end
+
     # The top-level workspace, see WorkSpace#main
     def main
       @workspace.main
@@ -473,28 +495,31 @@ module IRB
       @inspect_mode
     end
 
-    def evaluate(line, line_no, exception: nil) # :nodoc:
+    def evaluate(line, line_no) # :nodoc:
       @line_no = line_no
-      if exception
-        line_no -= 1
-        line = "begin ::Kernel.raise _; rescue _.class\n#{line}\n""end"
-        @workspace.local_variable_set(:_, exception)
-      end
+      result = nil
 
-      # Transform a non-identifier alias (@, $) or keywords (next, break)
-      command, args = line.split(/\s/, 2)
-      if original = command_aliases[command.to_sym]
-        line = line.gsub(/\A#{Regexp.escape(command)}/, original.to_s)
-        command = original
+      if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty?
+        IRB.set_measure_callback
       end
 
-      # Hook command-specific transformation
-      command_class = ExtendCommandBundle.load_command(command)
-      if command_class&.respond_to?(:transform_args)
-        line = "#{command} #{command_class.transform_args(args)}"
+      if IRB.conf[:MEASURE] && !IRB.conf[:MEASURE_CALLBACKS].empty?
+        last_proc = proc do
+          result = @workspace.evaluate(line, irb_path, line_no)
+        end
+        IRB.conf[:MEASURE_CALLBACKS].inject(last_proc) do |chain, item|
+          _name, callback, arg = item
+          proc do
+            callback.(self, line, line_no, arg) do
+              chain.call
+            end
+          end
+        end.call
+      else
+        result = @workspace.evaluate(line, irb_path, line_no)
       end
 
-      set_last_value(@workspace.evaluate(line, irb_path, line_no))
+      set_last_value(result)
     end
 
     def inspect_last_value # :nodoc:
@@ -551,5 +576,11 @@ module IRB
       command = command_aliases.fetch(command.to_sym, command)
       ExtendCommandBundle.load_command(command)&.respond_to?(:transform_args)
     end
+
+    def init_save_history# :nodoc:
+      unless (class<<@io;self;end).include?(HistorySavingAbility)
+        @io.extend(HistorySavingAbility)
+      end
+    end
   end
 end

data/lib/irb/ext/{history.rb → eval_history.rb}

@@ -40,14 +40,14 @@ module IRB # :nodoc:
     #
     # If +no+ is +nil+, execution result history isn't used (default).
     #
-    # History values are available via <code>__</code> variable, see
-    # IRB::History.
+    # EvalHistory values are available via <code>__</code> variable, see
+    # IRB::EvalHistory.
     def eval_history=(no)
       if no
         if defined?(@eval_history) && @eval_history
           @eval_history_values.size(no)
         else
-          @eval_history_values = History.new(no)
+          @eval_history_values = EvalHistory.new(no)
           IRB.conf[:__TMP__EHV__] = @eval_history_values
           @workspace.evaluate("__ = IRB.conf[:__TMP__EHV__]")
           IRB.conf.delete(:__TMP_EHV__)
@@ -89,7 +89,7 @@ module IRB # :nodoc:
   #    __[1]
   #    # => 10
   #
-  class History
+  class EvalHistory
 
     def initialize(size = 16)  # :nodoc:
       @size = size

data/lib/irb/ext/loader.rb

@@ -42,6 +42,7 @@ module IRB # :nodoc:
     #
     # See Irb#suspend_input_method for more information.
     def source_file(path)
+      irb = irb_context.irb
       irb.suspend_name(path, File.basename(path)) do
         FileInputMethod.open(path) do |io|
           irb.suspend_input_method(io) do
@@ -66,6 +67,7 @@ module IRB # :nodoc:
     #
     # See Irb#suspend_input_method for more information.
     def load_file(path, priv = nil)
+      irb = irb_context.irb
       irb.suspend_name(path, File.basename(path)) do
 
         if priv

data/lib/irb/extend-command.rb

@@ -289,7 +289,7 @@ module IRB # :nodoc:
           alias_method to, from
         }
       else
-        Kernel.print "irb: warn: can't alias #{to} from #{from}.\n"
+        Kernel.warn "irb: warn: can't alias #{to} from #{from}.\n"
       end
     end
 
@@ -316,10 +316,9 @@ module IRB # :nodoc:
     CE = ContextExtender # :nodoc:
 
     @EXTEND_COMMANDS = [
-      [:eval_history=, "ext/history.rb"],
+      [:eval_history=, "ext/eval_history.rb"],
       [:use_tracer=, "ext/tracer.rb"],
       [:use_loader=, "ext/use-loader.rb"],
-      [:save_history=, "ext/save-history.rb"],
     ]
 
     # Installs the default context extensions as irb commands:
@@ -327,7 +326,6 @@ module IRB # :nodoc:
     # Context#eval_history=::   +irb/ext/history.rb+
     # Context#use_tracer=::     +irb/ext/tracer.rb+
     # Context#use_loader=::     +irb/ext/use-loader.rb+
-    # Context#save_history=::   +irb/ext/save-history.rb+
     def self.install_extend_commands
       for args in @EXTEND_COMMANDS
         def_extend_command(*args)

data/lib/irb/{ext/save-history.rb → history.rb}

@@ -1,55 +1,4 @@
-# frozen_string_literal: false
-#
-#   save-history.rb -
-#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
-#
-
 module IRB
-  module HistorySavingAbility # :nodoc:
-  end
-
-  class Context
-    def init_save_history# :nodoc:
-      unless (class<<@io;self;end).include?(HistorySavingAbility)
-        @io.extend(HistorySavingAbility)
-      end
-    end
-
-    # A copy of the default <code>IRB.conf[:SAVE_HISTORY]</code>
-    def save_history
-      IRB.conf[:SAVE_HISTORY]
-    end
-
-    remove_method(:save_history=) if method_defined?(:save_history=)
-    # Sets <code>IRB.conf[:SAVE_HISTORY]</code> to the given +val+ and calls
-    # #init_save_history with this context.
-    #
-    # Will store the number of +val+ entries of history in the #history_file
-    #
-    # Add the following to your +.irbrc+ to change the number of history
-    # entries stored to 1000:
-    #
-    #     IRB.conf[:SAVE_HISTORY] = 1000
-    def save_history=(val)
-      IRB.conf[:SAVE_HISTORY] = val
-      if val
-        main_context = IRB.conf[:MAIN_CONTEXT]
-        main_context = self unless main_context
-        main_context.init_save_history
-      end
-    end
-
-    # A copy of the default <code>IRB.conf[:HISTORY_FILE]</code>
-    def history_file
-      IRB.conf[:HISTORY_FILE]
-    end
-
-    # Set <code>IRB.conf[:HISTORY_FILE]</code> to the given +hist+.
-    def history_file=(hist)
-      IRB.conf[:HISTORY_FILE] = hist
-    end
-  end
-
   module HistorySavingAbility # :nodoc:
     def HistorySavingAbility.extended(obj)
       IRB.conf[:AT_EXIT].push proc{obj.save_history}

data/lib/irb/input-method.rb

@@ -4,7 +4,6 @@
 #   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
 #
 
-require_relative 'src_encoding'
 require_relative 'completion'
 require 'io/console'
 require 'reline'
@@ -256,8 +255,6 @@ module IRB
   end
 
   class RelineInputMethod < InputMethod
-    include Reline
-
     # Creates a new input method object using Reline
     def initialize
       IRB.__send__(:set_encoding, Reline.encoding_system_needs.name, override: false)
@@ -270,9 +267,7 @@ module IRB
       @stdin = ::IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
       @stdout = ::IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-")
 
-      if Reline.respond_to?("basic_word_break_characters=")
-        Reline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
-      end
+      Reline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS
       Reline.completion_append_character = nil
       Reline.completer_quote_characters = ''
       Reline.completion_proc = IRB::InputCompletor::CompletionProc
@@ -401,10 +396,10 @@ module IRB
       mod_key = RUBY_PLATFORM.match?(/darwin/) ? "Option" : "Alt"
       message = "Press #{mod_key}+d to read the full document"
       contents = [message] + doc.accept(formatter).split("\n")
-      contents = contents.take(preferred_dialog_height) if respond_to?(:preferred_dialog_height)
+      contents = contents.take(Reline.preferred_dialog_height)
 
       y = cursor_pos_to_render.y
-      DialogRenderInfo.new(pos: Reline::CursorPos.new(x, y), contents: contents, width: width, bg_color: '49')
+      Reline::DialogRenderInfo.new(pos: Reline::CursorPos.new(x, y), contents: contents, width: width, bg_color: '49')
     }
 
     # Reads the next line from this input method.
@@ -415,8 +410,8 @@ module IRB
       Reline.output = @stdout
       Reline.prompt_proc = @prompt_proc
       Reline.auto_indent_proc = @auto_indent_proc if @auto_indent_proc
-      if l = readmultiline(@prompt, false, &@check_termination_proc)
-        HISTORY.push(l) if !l.empty?
+      if l = Reline.readmultiline(@prompt, false, &@check_termination_proc)
+        Reline::HISTORY.push(l) if !l.empty?
         @line[@line_no += 1] = l + "\n"
       else
         @eof = true
@@ -458,11 +453,7 @@ module IRB
     def inspect
       config = Reline::Config.new
       str = "RelineInputMethod with Reline #{Reline::VERSION}"
-      if config.respond_to?(:inputrc_path)
-        inputrc_path = File.expand_path(config.inputrc_path)
-      else
-        inputrc_path = File.expand_path(ENV['INPUTRC'] || '~/.inputrc')
-      end
+      inputrc_path = File.expand_path(config.inputrc_path)
       str += " and #{inputrc_path}" if File.exist?(inputrc_path)
       str
     end

data/lib/irb/nesting_parser.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+module IRB
+  module NestingParser
+    IGNORE_TOKENS = %i[on_sp on_ignored_nl on_comment on_embdoc_beg on_embdoc on_embdoc_end]
+
+    # Scan each token and call the given block with array of token and other information for parsing
+    def self.scan_opens(tokens)
+      opens = []
+      pending_heredocs = []
+      first_token_on_line = true
+      tokens.each do |t|
+        skip = false
+        last_tok, state, args = opens.last
+        case state
+        when :in_unquoted_symbol
+          unless IGNORE_TOKENS.include?(t.event)
+            opens.pop
+            skip = true
+          end
+        when :in_lambda_head
+          opens.pop if t.event == :on_tlambeg || (t.event == :on_kw && t.tok == 'do')
+        when :in_method_head
+          unless IGNORE_TOKENS.include?(t.event)
+            next_args = []
+            body = nil
+            if args.include?(:receiver)
+              case t.event
+              when :on_lparen, :on_ivar, :on_gvar, :on_cvar
+                # def (receiver). | def @ivar. | def $gvar. | def @@cvar.
+                next_args << :dot
+              when :on_kw
+                case t.tok
+                when 'self', 'true', 'false', 'nil'
+                  # def self(arg) | def self.
+                  next_args.push(:arg, :dot)
+                else
+                  # def if(arg)
+                  skip = true
+                  next_args << :arg
+                end
+              when :on_op, :on_backtick
+                # def +(arg)
+                skip = true
+                next_args << :arg
+              when :on_ident, :on_const
+                # def a(arg) | def a.
+                next_args.push(:arg, :dot)
+              end
+            end
+            if args.include?(:dot)
+              # def receiver.name
+              next_args << :name if t.event == :on_period || (t.event == :on_op && t.tok == '::')
+            end
+            if args.include?(:name)
+              if %i[on_ident on_const on_op on_kw on_backtick].include?(t.event)
+                # def name(arg) | def receiver.name(arg)
+                next_args << :arg
+                skip = true
+              end
+            end
+            if args.include?(:arg)
+              case t.event
+              when :on_nl, :on_semicolon
+                # def recever.f;
+                body = :normal
+              when :on_lparen
+                # def recever.f()
+                next_args << :eq
+              else
+                if t.event == :on_op && t.tok == '='
+                  # def receiver.f =
+                  body = :oneliner
+                else
+                  # def recever.f arg
+                  next_args << :arg_without_paren
+                end
+              end
+            end
+            if args.include?(:eq)
+              if t.event == :on_op && t.tok == '='
+                body = :oneliner
+              else
+                body = :normal
+              end
+            end
+            if args.include?(:arg_without_paren)
+              if %i[on_semicolon on_nl].include?(t.event)
+                # def f a;
+                body = :normal
+              else
+                # def f a, b
+                next_args << :arg_without_paren
+              end
+            end
+            if body == :oneliner
+              opens.pop
+            elsif body
+              opens[-1] = [last_tok, nil]
+            else
+              opens[-1] = [last_tok, :in_method_head, next_args]
+            end
+          end
+        when :in_for_while_until_condition
+          if t.event == :on_semicolon || t.event == :on_nl || (t.event == :on_kw && t.tok == 'do')
+            skip = true if t.event == :on_kw && t.tok == 'do'
+            opens[-1] = [last_tok, nil]
+          end
+        end
+
+        unless skip
+          case t.event
+          when :on_kw
+            case t.tok
+            when 'begin', 'class', 'module', 'do', 'case'
+              opens << [t, nil]
+            when 'end'
+              opens.pop
+            when 'def'
+              opens << [t, :in_method_head, [:receiver, :name]]
+            when 'if', 'unless'
+              unless t.state.allbits?(Ripper::EXPR_LABEL)
+                opens << [t, nil]
+              end
+            when 'while', 'until'
+              unless t.state.allbits?(Ripper::EXPR_LABEL)
+                opens << [t, :in_for_while_until_condition]
+              end
+            when 'ensure', 'rescue'
+              unless t.state.allbits?(Ripper::EXPR_LABEL)
+                opens.pop
+                opens << [t, nil]
+              end
+            when 'elsif', 'else', 'when'
+              opens.pop
+              opens << [t, nil]
+            when 'for'
+              opens << [t, :in_for_while_until_condition]
+            when 'in'
+              if last_tok&.event == :on_kw && %w[case in].include?(last_tok.tok) && first_token_on_line
+                opens.pop
+                opens << [t, nil]
+              end
+            end
+          when :on_tlambda
+            opens << [t, :in_lambda_head]
+          when :on_lparen, :on_lbracket, :on_lbrace, :on_tlambeg, :on_embexpr_beg, :on_embdoc_beg
+            opens << [t, nil]
+          when :on_rparen, :on_rbracket, :on_rbrace, :on_embexpr_end, :on_embdoc_end
+            opens.pop
+          when :on_heredoc_beg
+            pending_heredocs << t
+          when :on_heredoc_end
+            opens.pop
+          when :on_backtick
+            opens << [t, nil] if t.state.allbits?(Ripper::EXPR_BEG)
+          when :on_tstring_beg, :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_regexp_beg
+            opens << [t, nil]
+          when :on_tstring_end, :on_regexp_end, :on_label_end
+            opens.pop
+          when :on_symbeg
+            if t.tok == ':'
+              opens << [t, :in_unquoted_symbol]
+            else
+              opens << [t, nil]
+            end
+          end
+        end
+        if t.event == :on_nl || t.event == :on_semicolon
+          first_token_on_line = true
+        elsif t.event != :on_sp
+          first_token_on_line = false
+        end
+        if pending_heredocs.any? && t.tok.include?("\n")
+          pending_heredocs.reverse_each { |t| opens << [t, nil] }
+          pending_heredocs = []
+        end
+        yield t, opens if block_given?
+      end
+      opens.map(&:first) + pending_heredocs.reverse
+    end
+
+    def self.open_tokens(tokens)
+      # scan_opens without block will return a list of open tokens at last token position
+      scan_opens(tokens)
+    end
+
+    # Calculates token information [line_tokens, prev_opens, next_opens, min_depth] for each line.
+    # Example code
+    #   ["hello
+    #   world"+(
+    # First line
+    #   line_tokens: [[lbracket, '['], [tstring_beg, '"'], [tstring_content("hello\nworld"), "hello\n"]]
+    #   prev_opens:  []
+    #   next_tokens: [lbracket, tstring_beg]
+    #   min_depth:   0 (minimum at beginning of line)
+    # Second line
+    #   line_tokens: [[tstring_content("hello\nworld"), "world"], [tstring_end, '"'], [op, '+'], [lparen, '(']]
+    #   prev_opens:  [lbracket, tstring_beg]
+    #   next_tokens: [lbracket, lparen]
+    #   min_depth:   1 (minimum just after tstring_end)
+    def self.parse_by_line(tokens)
+      line_tokens = []
+      prev_opens = []
+      min_depth = 0
+      output = []
+      last_opens = scan_opens(tokens) do |t, opens|
+        depth = t == opens.last&.first ? opens.size - 1 : opens.size
+        min_depth = depth if depth < min_depth
+        if t.tok.include?("\n")
+          t.tok.each_line do |line|
+            line_tokens << [t, line]
+            next if line[-1] != "\n"
+            next_opens = opens.map(&:first)
+            output << [line_tokens, prev_opens, next_opens, min_depth]
+            prev_opens = next_opens
+            min_depth = prev_opens.size
+            line_tokens = []
+          end
+        else
+          line_tokens << [t, t.tok]
+        end
+      end
+      output << [line_tokens, prev_opens, last_opens, min_depth] if line_tokens.any?
+      output
+    end
+  end
+end