irb 1.1.0.pre.4

data/lib/irb/magic-file.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: false
+module IRB
+  class << (MagicFile = Object.new)
+    # see parser_magic_comment in parse.y
+    ENCODING_SPEC_RE = %r"coding\s*[=:]\s*([[:alnum:]\-_]+)"
+
+    def open(path)
+      io = File.open(path, 'rb')
+      line = io.gets
+      line = io.gets if line[0,2] == "#!"
+      encoding = detect_encoding(line)
+      internal_encoding = encoding
+      encoding ||= IRB.default_src_encoding
+      io.rewind
+      io.set_encoding(encoding, internal_encoding)
+
+      if block_given?
+        begin
+          return (yield io)
+        ensure
+          io.close
+        end
+      else
+        return io
+      end
+    end
+
+    private
+    def detect_encoding(line)
+      return unless line[0] == ?#
+      line = line[1..-1]
+      line = $1 if line[/-\*-\s*(.*?)\s*-*-$/]
+      return nil unless ENCODING_SPEC_RE =~ line
+      encoding = $1
+      return encoding.sub(/-(?:mac|dos|unix)/i, '')
+    end
+  end
+end

data/lib/irb/notifier.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: false
+#
+#   notifier.rb - output methods used by irb
+#   	$Release Version: 0.9.6$
+#   	$Revision$
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+# --
+#
+#
+#
+
+require "e2mmap"
+require_relative "output-method"
+
+module IRB
+  # An output formatter used internally by the lexer.
+  module Notifier
+    extend Exception2MessageMapper
+    def_exception :ErrUndefinedNotifier,
+      "undefined notifier level: %d is specified"
+    def_exception :ErrUnrecognizedLevel,
+      "unrecognized notifier level: %s is specified"
+
+    # Define a new Notifier output source, returning a new CompositeNotifier
+    # with the given +prefix+ and +output_method+.
+    #
+    # The optional +prefix+ will be appended to all objects being inspected
+    # during output, using the given +output_method+ as the output source. If
+    # no +output_method+ is given, StdioOutputMethod will be used, and all
+    # expressions will be sent directly to STDOUT without any additional
+    # formatting.
+    def def_notifier(prefix = "", output_method = StdioOutputMethod.new)
+      CompositeNotifier.new(prefix, output_method)
+    end
+    module_function :def_notifier
+
+    # An abstract class, or superclass, for CompositeNotifier and
+    # LeveledNotifier to inherit. It provides several wrapper methods for the
+    # OutputMethod object used by the Notifier.
+    class AbstractNotifier
+      # Creates a new Notifier object
+      def initialize(prefix, base_notifier)
+        @prefix = prefix
+        @base_notifier = base_notifier
+      end
+
+      # The +prefix+ for this Notifier, which is appended to all objects being
+      # inspected during output.
+      attr_reader :prefix
+
+      # A wrapper method used to determine whether notifications are enabled.
+      #
+      # Defaults to +true+.
+      def notify?
+        true
+      end
+
+      # See OutputMethod#print for more detail.
+      def print(*opts)
+        @base_notifier.print prefix, *opts if notify?
+      end
+
+      # See OutputMethod#printn for more detail.
+      def printn(*opts)
+        @base_notifier.printn prefix, *opts if notify?
+      end
+
+      # See OutputMethod#printf for more detail.
+      def printf(format, *opts)
+        @base_notifier.printf(prefix + format, *opts) if notify?
+      end
+
+      # See OutputMethod#puts for more detail.
+      def puts(*objs)
+        if notify?
+          @base_notifier.puts(*objs.collect{|obj| prefix + obj.to_s})
+        end
+      end
+
+      # Same as #ppx, except it uses the #prefix given during object
+      # initialization.
+      # See OutputMethod#ppx for more detail.
+      def pp(*objs)
+        if notify?
+          @base_notifier.ppx @prefix, *objs
+        end
+      end
+
+      # Same as #pp, except it concatenates the given +prefix+ with the #prefix
+      # given during object initialization.
+      #
+      # See OutputMethod#ppx for more detail.
+      def ppx(prefix, *objs)
+        if notify?
+          @base_notifier.ppx @prefix+prefix, *objs
+        end
+      end
+
+      # Execute the given block if notifications are enabled.
+      def exec_if
+        yield(@base_notifier) if notify?
+      end
+    end
+
+    # A class that can be used to create a group of notifier objects with the
+    # intent of representing a leveled notification system for irb.
+    #
+    # This class will allow you to generate other notifiers, and assign them
+    # the appropriate level for output.
+    #
+    # The Notifier class provides a class-method Notifier.def_notifier to
+    # create a new composite notifier. Using the first composite notifier
+    # object you create, sibling notifiers can be initialized with
+    # #def_notifier.
+    class CompositeNotifier < AbstractNotifier
+      # Create a new composite notifier object with the given +prefix+, and
+      # +base_notifier+ to use for output.
+      def initialize(prefix, base_notifier)
+        super
+
+        @notifiers = [D_NOMSG]
+        @level_notifier = D_NOMSG
+      end
+
+      # List of notifiers in the group
+      attr_reader :notifiers
+
+      # Creates a new LeveledNotifier in the composite #notifiers group.
+      #
+      # The given +prefix+ will be assigned to the notifier, and +level+ will
+      # be used as the index of the #notifiers Array.
+      #
+      # This method returns the newly created instance.
+      def def_notifier(level, prefix = "")
+        notifier = LeveledNotifier.new(self, level, prefix)
+        @notifiers[level] = notifier
+        notifier
+      end
+
+      # Returns the leveled notifier for this object
+      attr_reader :level_notifier
+      alias level level_notifier
+
+      # Sets the leveled notifier for this object.
+      #
+      # When the given +value+ is an instance of AbstractNotifier,
+      # #level_notifier is set to the given object.
+      #
+      # When an Integer is given, #level_notifier is set to the notifier at the
+      # index +value+ in the #notifiers Array.
+      #
+      # If no notifier exists at the index +value+ in the #notifiers Array, an
+      # ErrUndefinedNotifier exception is raised.
+      #
+      # An ErrUnrecognizedLevel exception is raised if the given +value+ is not
+      # found in the existing #notifiers Array, or an instance of
+      # AbstractNotifier
+      def level_notifier=(value)
+        case value
+        when AbstractNotifier
+          @level_notifier = value
+        when Integer
+          l = @notifiers[value]
+          Notifier.Raise ErrUndefinedNotifier, value unless l
+          @level_notifier = l
+        else
+          Notifier.Raise ErrUnrecognizedLevel, value unless l
+        end
+      end
+
+      alias level= level_notifier=
+    end
+
+    # A leveled notifier is comparable to the composite group from
+    # CompositeNotifier#notifiers.
+    class LeveledNotifier < AbstractNotifier
+      include Comparable
+
+      # Create a new leveled notifier with the given +base+, and +prefix+ to
+      # send to AbstractNotifier.new
+      #
+      # The given +level+ is used to compare other leveled notifiers in the
+      # CompositeNotifier group to determine whether or not to output
+      # notifications.
+      def initialize(base, level, prefix)
+        super(prefix, base)
+
+        @level = level
+      end
+
+      # The current level of this notifier object
+      attr_reader :level
+
+      # Compares the level of this notifier object with the given +other+
+      # notifier.
+      #
+      # See the Comparable module for more information.
+      def <=>(other)
+        @level <=> other.level
+      end
+
+      # Whether to output messages to the output method, depending on the level
+      # of this notifier object.
+      def notify?
+        @base_notifier.level >= self
+      end
+    end
+
+    # NoMsgNotifier is a LeveledNotifier that's used as the default notifier
+    # when creating a new CompositeNotifier.
+    #
+    # This notifier is used as the +zero+ index, or level +0+, for
+    # CompositeNotifier#notifiers, and will not output messages of any sort.
+    class NoMsgNotifier < LeveledNotifier
+      # Creates a new notifier that should not be used to output messages.
+      def initialize
+        @base_notifier = nil
+        @level = 0
+        @prefix = ""
+      end
+
+      # Ensures notifications are ignored, see AbstractNotifier#notify? for
+      # more information.
+      def notify?
+        false
+      end
+    end
+
+    D_NOMSG = NoMsgNotifier.new # :nodoc:
+  end
+end

data/lib/irb/output-method.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: false
+#
+#   output-method.rb - output methods used by irb
+#   	$Release Version: 0.9.6$
+#   	$Revision$
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+# --
+#
+#
+#
+
+require "e2mmap"
+
+module IRB
+  # An abstract output class for IO in irb. This is mainly used internally by
+  # IRB::Notifier. You can define your own output method to use with Irb.new,
+  # or Context.new
+  class OutputMethod
+    extend Exception2MessageMapper
+    def_exception :NotImplementedError, "Need to define `%s'"
+
+
+    # Open this method to implement your own output method, raises a
+    # NotImplementedError if you don't define #print in your own class.
+    def print(*opts)
+      OutputMethod.Raise NotImplementedError, "print"
+    end
+
+    # Prints the given +opts+, with a newline delimiter.
+    def printn(*opts)
+      print opts.join(" "), "\n"
+    end
+
+    # Extends IO#printf to format the given +opts+ for Kernel#sprintf using
+    # #parse_printf_format
+    def printf(format, *opts)
+      if /(%*)%I/ =~ format
+        format, opts = parse_printf_format(format, opts)
+      end
+      print sprintf(format, *opts)
+    end
+
+    # Returns an array of the given +format+ and +opts+ to be used by
+    # Kernel#sprintf, if there was a successful Regexp match in the given
+    # +format+ from #printf
+    #
+    #     %
+    #     <flag>  [#0- +]
+    #     <minimum field width> (\*|\*[1-9][0-9]*\$|[1-9][0-9]*)
+    #     <precision>.(\*|\*[1-9][0-9]*\$|[1-9][0-9]*|)?
+    #     #<length modifier>(hh|h|l|ll|L|q|j|z|t)
+    #     <conversion specifier>[diouxXeEfgGcsb%]
+    def parse_printf_format(format, opts)
+      return format, opts if $1.size % 2 == 1
+    end
+
+    # Calls #print on each element in the given +objs+, followed by a newline
+    # character.
+    def puts(*objs)
+      for obj in objs
+        print(*obj)
+        print "\n"
+      end
+    end
+
+    # Prints the given +objs+ calling Object#inspect on each.
+    #
+    # See #puts for more detail.
+    def pp(*objs)
+      puts(*objs.collect{|obj| obj.inspect})
+    end
+
+    # Prints the given +objs+ calling Object#inspect on each and appending the
+    # given +prefix+.
+    #
+    # See #puts for more detail.
+    def ppx(prefix, *objs)
+      puts(*objs.collect{|obj| prefix+obj.inspect})
+    end
+
+  end
+
+  # A standard output printer
+  class StdioOutputMethod < OutputMethod
+    # Prints the given +opts+ to standard output, see IO#print for more
+    # information.
+    def print(*opts)
+      STDOUT.print(*opts)
+    end
+  end
+end

data/lib/irb/ruby-lex.rb

@@ -0,0 +1,499 @@
+# frozen_string_literal: false
+#
+#   irb/ruby-lex.rb - ruby lexcal analyzer
+#   	$Release Version: 0.9.6$
+#   	$Revision$
+#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
+#
+# --
+#
+#
+#
+
+require "e2mmap"
+require "ripper"
+
+# :stopdoc:
+class RubyLex
+
+  extend Exception2MessageMapper
+  def_exception(:TerminateLineInput, "Terminate Line Input")
+
+  def initialize
+    @exp_line_no = @line_no = 1
+    @indent = 0
+    @continue = false
+    @line = ""
+    @prompt = nil
+  end
+
+  # io functions
+  def set_input(io, p = nil, &block)
+    @io = io
+    if @io.respond_to?(:check_termination)
+      @io.check_termination do |code|
+        code.gsub!(/\s*\z/, '').concat("\n")
+        ltype, indent, continue, code_block_open = check_state(code)
+        if ltype or indent > 0 or continue or code_block_open
+          false
+        else
+          true
+        end
+      end
+    end
+    if @io.respond_to?(:dynamic_prompt)
+      @io.dynamic_prompt do |lines|
+        lines << '' if lines.empty?
+        result = []
+        lines.each_index { |i|
+          c = lines[0..i].map{ |l| l + "\n" }.join
+          ltype, indent, continue, code_block_open = check_state(c)
+          result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + i)
+        }
+        result
+      end
+    end
+    if p.respond_to?(:call)
+      @input = p
+    elsif block_given?
+      @input = block
+    else
+      @input = Proc.new{@io.gets}
+    end
+  end
+
+  def set_prompt(p = nil, &block)
+    p = block if block_given?
+    if p.respond_to?(:call)
+      @prompt = p
+    else
+      @prompt = Proc.new{print p}
+    end
+  end
+
+  def ripper_lex_without_warning(code)
+    verbose, $VERBOSE = $VERBOSE, nil
+    tokens = Ripper.lex(code)
+    $VERBOSE = verbose
+    tokens
+  end
+
+  def set_auto_indent(context)
+    if @io.respond_to?(:auto_indent) and context.auto_indent_mode
+      @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
+        if is_newline
+          md = lines[line_index - 1].match(/(\A +)/)
+          prev_spaces = md.nil? ? 0 : md[1].count(' ')
+          @tokens = ripper_lex_without_warning(lines[0..line_index].join("\n"))
+          depth_difference = check_newline_depth_difference
+          prev_spaces + depth_difference * 2
+        else
+          code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
+          last_line = lines[line_index]&.byteslice(0, byte_pointer)
+          code += last_line if last_line
+          @tokens = ripper_lex_without_warning(code)
+          corresponding_token_depth = check_corresponding_token_depth
+          if corresponding_token_depth
+            corresponding_token_depth
+          else
+            nil
+          end
+        end
+      end
+    end
+  end
+
+  def check_state(code)
+    @tokens = ripper_lex_without_warning(code)
+    ltype = process_literal_type
+    indent = process_nesting_level
+    continue = process_continue
+    code_block_open = check_code_block(code)
+    [ltype, indent, continue, code_block_open]
+  end
+
+  def prompt
+    if @prompt
+      @prompt.call(@ltype, @indent, @continue, @line_no)
+    end
+  end
+
+  def initialize_input
+    @ltype = nil
+    @indent = 0
+    @continue = false
+    @line = ""
+    @exp_line_no = @line_no
+    @code_block_open = false
+  end
+
+  def each_top_level_statement
+    initialize_input
+    catch(:TERM_INPUT) do
+      loop do
+        begin
+          prompt
+          unless l = lex
+            throw :TERM_INPUT if @line == ''
+          else
+            @line_no += l.count("\n")
+            next if l == "\n"
+            @line.concat l
+            if @code_block_open or @ltype or @continue or @indent > 0
+              next
+            end
+          end
+          if @line != "\n"
+            @line.force_encoding(@io.encoding)
+            yield @line, @exp_line_no
+          end
+          break if @io.eof?
+          @line = ''
+          @exp_line_no = @line_no
+
+          @indent = 0
+        rescue TerminateLineInput
+          initialize_input
+          prompt
+        end
+      end
+    end
+  end
+
+  def lex
+    line = @input.call
+    if @io.respond_to?(:check_termination)
+      return line # multiline
+    end
+    code = @line + (line.nil? ? '' : line)
+    code.gsub!(/\s*\z/, '').concat("\n")
+    @tokens = ripper_lex_without_warning(code)
+    @continue = process_continue
+    @code_block_open = check_code_block(code)
+    @indent = process_nesting_level
+    @ltype = process_literal_type
+    line
+  end
+
+  def process_continue
+    # last token is always newline
+    if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end
+      # end of regexp literal
+      return false
+    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon
+      return false
+    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(@tokens[-2][2])
+      return false
+    elsif @tokens.size >= 3 and @tokens[-3][1] == :on_symbeg and @tokens[-2][1] == :on_ivar
+      # This is for :@a or :@1 because :@1 ends with EXPR_FNAME
+      return false
+    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_ivar and @tokens[-2][2] =~ /\A@\d+\z/
+      # This is for @1
+      return false
+    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_cvar and @tokens[-1][1] == :on_int
+      # This is for @@1 or :@@1 and ends with on_int because it's syntax error
+      return false
+    elsif !@tokens.empty? and @tokens.last[2] == "\\\n"
+      return true
+    elsif @tokens.size >= 1 and @tokens[-1][1] == :on_heredoc_end # "EOH\n"
+      return false
+    elsif @tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and @tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME)
+      # end of literal except for regexp
+      return true
+    end
+    false
+  end
+
+  def check_code_block(code)
+    return true if @tokens.empty?
+    if @tokens.last[1] == :on_heredoc_beg
+      return true
+    end
+
+    begin # check if parser error are available
+      verbose, $VERBOSE = $VERBOSE, nil
+      case RUBY_ENGINE
+      when 'jruby'
+        JRuby.compile_ir(code)
+      else
+        RubyVM::InstructionSequence.compile(code)
+      end
+    rescue SyntaxError => e
+      case e.message
+      when /unterminated (?:string|regexp) meets end of file/
+        # "unterminated regexp meets end of file"
+        #
+        #   example:
+        #     /
+        #
+        # "unterminated string meets end of file"
+        #
+        #   example:
+        #     '
+        return true
+      when /syntax error, unexpected end-of-input/
+        # "syntax error, unexpected end-of-input, expecting keyword_end"
+        #
+        #   example:
+        #     if ture
+        #       hoge
+        #       if false
+        #         fuga
+        #       end
+        return true
+      when /syntax error, unexpected keyword_end/
+        # "syntax error, unexpected keyword_end"
+        #
+        #   example:
+        #     if (
+        #     end
+        #
+        #   example:
+        #     end
+        return false
+      when /syntax error, unexpected '\.'/
+        # "syntax error, unexpected '.'"
+        #
+        #   example:
+        #     .
+        return false
+      when /unexpected tREGEXP_BEG/
+        # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
+        #
+        #   example:
+        #     method / f /
+        return false
+      when /numbered parameter outside block/
+        # "numbered parameter outside block"
+        #
+        #   example:
+        #     :@1
+        return false
+      end
+    ensure
+      $VERBOSE = verbose
+    end
+
+    if defined?(Ripper::EXPR_BEG)
+      last_lex_state = @tokens.last[3]
+      if last_lex_state.allbits?(Ripper::EXPR_BEG)
+        return false
+      elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
+        return true
+      elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
+        return true
+      elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
+        return true
+      elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
+        return true
+      elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
+        return false
+      end
+    end
+
+    false
+  end
+
+  def process_nesting_level
+    indent = 0
+    @tokens.each_with_index { |t, index|
+      case t[1]
+      when :on_lbracket, :on_lbrace, :on_lparen
+        indent += 1
+      when :on_rbracket, :on_rbrace, :on_rparen
+        indent -= 1
+      when :on_kw
+        next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
+        case t[2]
+        when 'do'
+          if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_CMDARG)
+            # method_with_bock do; end
+            indent += 1
+          else
+            # while cond do; end # also "until" or "for"
+            # This "do" doesn't increment indent because "while" already
+            # incremented.
+          end
+        when 'def', 'case', 'for', 'begin', 'class', 'module'
+          indent += 1
+        when 'if', 'unless', 'while', 'until'
+          # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
+          indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
+        when 'end'
+          indent -= 1
+        end
+      end
+      # percent literals are not indented
+    }
+    indent
+  end
+
+  def check_newline_depth_difference
+    depth_difference = 0
+    @tokens.each_with_index do |t, index|
+      case t[1]
+      when :on_ignored_nl, :on_nl
+        if index != (@tokens.size - 1)
+          depth_difference = 0
+        end
+        next
+      when :on_sp
+        next
+      end
+      case t[1]
+      when :on_lbracket, :on_lbrace, :on_lparen
+        depth_difference += 1
+      when :on_rbracket, :on_rbrace, :on_rparen
+        depth_difference -= 1
+      when :on_kw
+        next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
+        case t[2]
+        when 'do'
+          if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_CMDARG)
+            # method_with_bock do; end
+            depth_difference += 1
+          else
+            # while cond do; end # also "until" or "for"
+            # This "do" doesn't increment indent because "while" already
+            # incremented.
+          end
+        when 'def', 'case', 'for', 'begin', 'class', 'module'
+          depth_difference += 1
+        when 'if', 'unless', 'while', 'until'
+          # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
+          unless t[3].allbits?(Ripper::EXPR_LABEL)
+            depth_difference += 1
+          end
+        when 'else', 'elsif', 'rescue', 'ensure', 'when', 'in'
+          depth_difference += 1
+        end
+      end
+    end
+    depth_difference
+  end
+
+  def check_corresponding_token_depth
+    corresponding_token_depth = nil
+    is_first_spaces_of_line = true
+    is_first_printable_of_line = true
+    spaces_of_nest = []
+    spaces_at_line_head = 0
+    @tokens.each_with_index do |t, index|
+      corresponding_token_depth = nil
+      case t[1]
+      when :on_ignored_nl, :on_nl
+        spaces_at_line_head = 0
+        is_first_spaces_of_line = true
+        is_first_printable_of_line = true
+        next
+      when :on_sp
+        spaces_at_line_head = t[2].count(' ') if is_first_spaces_of_line
+        is_first_spaces_of_line = false
+        next
+      end
+      case t[1]
+      when :on_lbracket, :on_lbrace, :on_lparen
+        spaces_of_nest.push(spaces_at_line_head)
+      when :on_rbracket, :on_rbrace, :on_rparen
+        if is_first_printable_of_line
+          corresponding_token_depth = spaces_of_nest.pop
+        else
+          spaces_of_nest.pop
+          corresponding_token_depth = nil
+        end
+      when :on_kw
+        next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
+        case t[2]
+        when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
+          spaces_of_nest.push(spaces_at_line_head)
+        when 'if', 'unless', 'while', 'until'
+          # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
+          unless t[3].allbits?(Ripper::EXPR_LABEL)
+            spaces_of_nest.push(spaces_at_line_head)
+          end
+        when 'else', 'elsif', 'rescue', 'ensure', 'when', 'in'
+          corresponding_token_depth = spaces_of_nest.last
+        when 'end'
+          if is_first_printable_of_line
+            corresponding_token_depth = spaces_of_nest.pop
+          else
+            spaces_of_nest.pop
+            corresponding_token_depth = nil
+          end
+        end
+      end
+      is_first_spaces_of_line = false
+      is_first_printable_of_line = false
+    end
+    corresponding_token_depth
+  end
+
+  def check_string_literal
+    i = 0
+    start_token = []
+    end_type = []
+    while i < @tokens.size
+      t = @tokens[i]
+      case t[1]
+      when :on_tstring_beg
+        start_token << t
+        end_type << [:on_tstring_end, :on_label_end]
+      when :on_regexp_beg
+        start_token << t
+        end_type << :on_regexp_end
+      when :on_symbeg
+        acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw}
+        if (i + 1) < @tokens.size and acceptable_single_tokens.all?{ |t| @tokens[i + 1][1] != t }
+          start_token << t
+          end_type << :on_tstring_end
+        end
+      when :on_backtick
+        start_token << t
+        end_type << :on_tstring_end
+      when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
+        start_token << t
+        end_type << :on_tstring_end
+      when :on_heredoc_beg
+        start_token << t
+        end_type << :on_heredoc_end
+      when *end_type.last
+        start_token.pop
+        end_type.pop
+      end
+      i += 1
+    end
+    start_token.last.nil? ? '' : start_token.last
+  end
+
+  def process_literal_type
+    start_token = check_string_literal
+    case start_token[1]
+    when :on_tstring_beg
+      case start_token[2]
+      when ?"      then ?"
+      when /^%.$/  then ?"
+      when /^%Q.$/ then ?"
+      when ?'      then ?'
+      when /^%q.$/ then ?'
+      end
+    when :on_regexp_beg   then ?/
+    when :on_symbeg       then ?:
+    when :on_backtick     then ?`
+    when :on_qwords_beg   then ?]
+    when :on_words_beg    then ?]
+    when :on_qsymbols_beg then ?]
+    when :on_symbols_beg  then ?]
+    when :on_heredoc_beg
+      start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
+      case $1
+      when ?" then ?"
+      when ?' then ?'
+      when ?` then ?`
+      else         ?"
+      end
+    else
+      nil
+    end
+  end
+end
+# :startdoc: