rdoc 6.0.4 → 6.1.0.beta1

data/lib/rdoc/markup/heading.rb

@@ -23,12 +23,12 @@ RDoc::Markup::Heading =
     return @to_html if @to_html
 
     markup = RDoc::Markup.new
-    markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+    markup.add_regexp_handling RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
 
     @to_html = RDoc::Markup::ToHtml.new nil
 
-    def @to_html.handle_special_CROSSREF special
-      special.text.sub(/^\\/, '')
+    def @to_html.handle_regexp_CROSSREF target
+      target.text.sub(/^\\/, '')
     end
 
     @to_html

data/lib/rdoc/markup/to_bs.rb

@@ -41,7 +41,7 @@ class RDoc::Markup::ToBs < RDoc::Markup::ToRdoc
   end
 
   ##
-  # Turns on or off special handling for +convert_string+
+  # Turns on or off regexp handling for +convert_string+
 
   def annotate tag
     case tag
@@ -54,9 +54,9 @@ class RDoc::Markup::ToBs < RDoc::Markup::ToRdoc
   end
 
   ##
-  # Calls convert_string on the result of convert_special
+  # Calls convert_string on the result of convert_regexp_handling
 
-  def convert_special special
+  def convert_regexp_handling target
     convert_string super
   end
 

data/lib/rdoc/markup/to_html.rb

@@ -53,18 +53,18 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     @hard_break = "<br>\n"
 
     # external links
-    @markup.add_special(/(?:link:|https?:|mailto:|ftp:|irc:|www\.)\S+\w/,
-                        :HYPERLINK)
+    @markup.add_regexp_handling(/(?:link:|https?:|mailto:|ftp:|irc:|www\.)\S+\w/,
+                                :HYPERLINK)
 
-    add_special_RDOCLINK
-    add_special_TIDYLINK
+    add_regexp_handling_RDOCLINK
+    add_regexp_handling_TIDYLINK
 
     init_tags
   end
 
-  # :section: Special Handling
+  # :section: Regexp Handling
   #
-  # These methods handle special markup added by RDoc::Markup#add_special.
+  # These methods are used by regexp handling markup added by RDoc::Markup#add_regexp_handling.
 
   def handle_RDOCLINK url # :nodoc:
     case url
@@ -91,14 +91,14 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   end
 
   ##
-  # +special+ is a <code><br></code>
+  # +target+ is a <code><br></code>
 
-  def handle_special_HARD_BREAK special
+  def handle_regexp_HARD_BREAK target
     '<br>'
   end
 
   ##
-  # +special+ is a potential link.  The following schemes are handled:
+  # +target+ is a potential link.  The following schemes are handled:
   #
   # <tt>mailto:</tt>::
   #   Inserted as-is.
@@ -109,14 +109,14 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # <tt>link:</tt>::
   #   Reference to a local file relative to the output directory.
 
-  def handle_special_HYPERLINK(special)
-    url = special.text
+  def handle_regexp_HYPERLINK(target)
+    url = target.text
 
     gen_url url, url
   end
 
   ##
-  # +special+ is an rdoc-schemed link that will be converted into a hyperlink.
+  # +target+ is an rdoc-schemed link that will be converted into a hyperlink.
   #
   # For the +rdoc-ref+ scheme the named reference will be returned without
   # creating a link.
@@ -124,16 +124,16 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # For the +rdoc-label+ scheme the footnote and label prefixes are stripped
   # when creating a link.  All other contents will be linked verbatim.
 
-  def handle_special_RDOCLINK special
-    handle_RDOCLINK special.text
+  def handle_regexp_RDOCLINK target
+    handle_RDOCLINK target.text
   end
 
   ##
-  # This +special+ is a link where the label is different from the URL
+  # This +target+ is a link where the label is different from the URL
   # <tt>label[url]</tt> or <tt>{long label}[url]</tt>
 
-  def handle_special_TIDYLINK(special)
-    text = special.text
+  def handle_regexp_TIDYLINK(target)
+    text = target.text
 
     return text unless
       text =~ /^\{(.*)\}\[(.*?)\]$/ or text =~ /^(\S+)\[(.*?)\]$/
@@ -186,7 +186,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     @res << "\n<p>"
     text = paragraph.text @hard_break
     text = text.gsub(/\r?\n/, ' ')
-    @res << wrap(to_html(text))
+    @res << to_html(text)
     @res << "</p>\n"
   end
 
@@ -200,7 +200,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
     content = if verbatim.ruby? or parseable? text then
                 begin
-                  tokens = RDoc::RipperStateLex.parse text
+                  tokens = RDoc::Parser::RipperStateLex.parse text
                   klass  = ' class="ruby"'
 
                   result = RDoc::TokenStream.to_html tokens
@@ -312,7 +312,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
   ##
   # Generate a link to +url+ with content +text+.  Handles the special cases
-  # for img: and link: described under handle_special_HYPERLINK
+  # for img: and link: described under handle_regexp_HYPERLINK
 
   def gen_url url, text
     scheme, url, id = parse_url url

data/lib/rdoc/markup/to_html_crossref.rb

@@ -40,7 +40,7 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
     @show_hash     = @options.show_hash
 
     crossref_re = @hyperlink_all ? ALL_CROSSREF_REGEXP : CROSSREF_REGEXP
-    @markup.add_special crossref_re, :CROSSREF
+    @markup.add_regexp_handling crossref_re, :CROSSREF
 
     @cross_reference = RDoc::CrossReference.new @context
   end
@@ -68,8 +68,8 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # example, ToHtml is found, even without the <tt>RDoc::Markup::</tt> prefix,
   # because we look for it in module Markup first.
 
-  def handle_special_CROSSREF(special)
-    name = special.text
+  def handle_regexp_CROSSREF(target)
+    name = target.text
 
     return name if name =~ /@[\w-]+\.[\w-]/ # labels that look like emails
 
@@ -87,22 +87,22 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # Handles <tt>rdoc-ref:</tt> scheme links and allows RDoc::Markup::ToHtml to
   # handle other schemes.
 
-  def handle_special_HYPERLINK special
-    return cross_reference $' if special.text =~ /\Ardoc-ref:/
+  def handle_regexp_HYPERLINK target
+    return cross_reference $' if target.text =~ /\Ardoc-ref:/
 
     super
   end
 
   ##
-  # +special+ is an rdoc-schemed link that will be converted into a hyperlink.
+  # +target+ is an rdoc-schemed link that will be converted into a hyperlink.
   # For the rdoc-ref scheme the cross-reference will be looked up and the
   # given name will be used.
   #
   # All other contents are handled by
-  # {the superclass}[rdoc-ref:RDoc::Markup::ToHtml#handle_special_RDOCLINK]
+  # {the superclass}[rdoc-ref:RDoc::Markup::ToHtml#handle_regexp_RDOCLINK]
 
-  def handle_special_RDOCLINK special
-    url = special.text
+  def handle_regexp_RDOCLINK target
+    url = target.text
 
     case url
     when /\Ardoc-ref:/ then
@@ -126,8 +126,6 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # Creates an HTML link to +name+ with the given +text+.
 
   def link name, text
-    original_name = name
-
     if name =~ /(.*[^#:])@/ then
       name = $1
       label = $'
@@ -135,9 +133,6 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
 
     ref = @cross_reference.resolve name, text
 
-    text = ref.output_name @context if
-      RDoc::MethodAttr === ref and text == original_name
-
     case ref
     when String then
       ref

data/lib/rdoc/markup/to_html_snippet.rb

@@ -44,7 +44,7 @@ class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml
     @mask       = 0
     @paragraphs = 0
 
-    @markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+    @markup.add_regexp_handling RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
   end
 
   ##
@@ -71,7 +71,7 @@ class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml
 
     text = paragraph.text @hard_break
 
-    @res << "#{para}#{wrap to_html text}\n"
+    @res << "#{para}#{to_html text}\n"
 
     add_paragraph
   end
@@ -123,16 +123,16 @@ class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml
   end
 
   ##
-  # Removes escaping from the cross-references in +special+
+  # Removes escaping from the cross-references in +target+
 
-  def handle_special_CROSSREF special
-    special.text.sub(/\A\\/, '')
+  def handle_regexp_CROSSREF target
+    target.text.sub(/\A\\/, '')
   end
 
   ##
-  # +special+ is a <code><br></code>
+  # +target+ is a <code><br></code>
 
-  def handle_special_HARD_BREAK special
+  def handle_regexp_HARD_BREAK target
     @characters -= 4
     '<br>'
   end
@@ -226,8 +226,8 @@ class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml
       when String then
         text = convert_string item
         res << truncate(text)
-      when RDoc::Markup::Special then
-        text = convert_special item
+      when RDoc::Markup::RegexpHandling then
+        text = convert_regexp_handling item
         res << truncate(text)
       else
         raise "Unknown flow element: #{item.inspect}"

data/lib/rdoc/markup/to_label.rb

@@ -16,8 +16,8 @@ class RDoc::Markup::ToLabel < RDoc::Markup::Formatter
   def initialize markup = nil
     super nil, markup
 
-    @markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
-    @markup.add_special(/(((\{.*?\})|\b\S+?)\[\S+?\])/, :TIDYLINK)
+    @markup.add_regexp_handling RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+    @markup.add_regexp_handling(/(((\{.*?\})|\b\S+?)\[\S+?\])/, :TIDYLINK)
 
     add_tag :BOLD, '', ''
     add_tag :TT,   '', ''
@@ -36,20 +36,20 @@ class RDoc::Markup::ToLabel < RDoc::Markup::Formatter
   end
 
   ##
-  # Converts the CROSSREF +special+ to plain text, removing the suppression
+  # Converts the CROSSREF +target+ to plain text, removing the suppression
   # marker, if any
 
-  def handle_special_CROSSREF special
-    text = special.text
+  def handle_regexp_CROSSREF target
+    text = target.text
 
     text.sub(/^\\/, '')
   end
 
   ##
-  # Converts the TIDYLINK +special+ to just the text part
+  # Converts the TIDYLINK +target+ to just the text part
 
-  def handle_special_TIDYLINK special
-    text = special.text
+  def handle_regexp_TIDYLINK target
+    text = target.text
 
     return text unless text =~ /\{(.*?)\}\[(.*?)\]/ or text =~ /(\S+)\[(.*?)\]/
 
@@ -68,7 +68,7 @@ class RDoc::Markup::ToLabel < RDoc::Markup::Formatter
   alias accept_rule               ignore
   alias accept_verbatim           ignore
   alias end_accepting             ignore
-  alias handle_special_HARD_BREAK ignore
+  alias handle_regexp_HARD_BREAK  ignore
   alias start_accepting           ignore
 
 end

data/lib/rdoc/markup/to_markdown.rb

@@ -19,8 +19,8 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc
     @headings[5] = ['##### ',  '']
     @headings[6] = ['###### ', '']
 
-    add_special_RDOCLINK
-    add_special_TIDYLINK
+    add_regexp_handling_RDOCLINK
+    add_regexp_handling_TIDYLINK
 
     @hard_break = "  \n"
   end
@@ -37,7 +37,7 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc
   ##
   # Adds a newline to the output
 
-  def handle_special_HARD_BREAK special
+  def handle_regexp_HARD_BREAK target
     "  \n"
   end
 
@@ -166,8 +166,8 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc
   ##
   # Converts the RDoc markup tidylink into a Markdown.style link.
 
-  def handle_special_TIDYLINK special
-    text = special.text
+  def handle_regexp_TIDYLINK target
+    text = target.text
 
     return text unless text =~ /\{(.*?)\}\[(.*?)\]/ or text =~ /(\S+)\[(.*?)\]/
 
@@ -184,8 +184,8 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc
   ##
   # Converts the rdoc-...: links into a Markdown.style links.
 
-  def handle_special_RDOCLINK special
-    handle_rdoc_link special.text
+  def handle_regexp_RDOCLINK target
+    handle_rdoc_link target.text
   end
 
 end

data/lib/rdoc/markup/to_rdoc.rb

@@ -45,7 +45,7 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   def initialize markup = nil
     super nil, markup
 
-    @markup.add_special(/\\\S/, :SUPPRESSED_CROSSREF)
+    @markup.add_regexp_handling(/\\\S/, :SUPPRESSED_CROSSREF)
     @width = 78
     init_tags
 
@@ -253,10 +253,10 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   end
 
   ##
-  # Removes preceding \\ from the suppressed crossref +special+
+  # Removes preceding \\ from the suppressed crossref +target+
 
-  def handle_special_SUPPRESSED_CROSSREF special
-    text = special.text
+  def handle_regexp_SUPPRESSED_CROSSREF target
+    text = target.text
     text = text.sub('\\', '') unless in_tt?
     text
   end
@@ -264,7 +264,7 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   ##
   # Adds a newline to the output
 
-  def handle_special_HARD_BREAK special
+  def handle_regexp_HARD_BREAK target
     "\n"
   end
 

data/lib/rdoc/markup/to_tt_only.rb

@@ -91,8 +91,8 @@ class RDoc::Markup::ToTtOnly < RDoc::Markup::Formatter
       when RDoc::Markup::AttrChanger then
         off_tags res, item
         on_tags res, item
-      when RDoc::Markup::Special then
-        @res << convert_special(item) if in_tt? # TODO can this happen?
+      when RDoc::Markup::RegexpHandling then
+        @res << convert_regexp_handling(item) if in_tt? # TODO can this happen?
       else
         raise "Unknown flow element: #{item.inspect}"
       end

data/lib/rdoc/parser/ripper_state_lex.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 require 'ripper'
 
-class RDoc::RipperStateLex
+class RDoc::Parser::RipperStateLex
   # TODO: Remove this constants after Ruby 2.4 EOL
   RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)
 
+  Token = Struct.new(:line_no, :char_no, :kind, :text, :state)
+
   EXPR_NONE = 0
   EXPR_BEG = 1
   EXPR_END = 2
@@ -48,7 +50,7 @@ class RDoc::RipperStateLex
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_nl(tok, data)
@@ -59,7 +61,7 @@ class RDoc::RipperStateLex
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_op(tok, data)
@@ -101,7 +103,7 @@ class RDoc::RipperStateLex
           @lex_state = EXPR_BEG
         end
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_kw(tok, data)
@@ -114,7 +116,7 @@ class RDoc::RipperStateLex
         @continue = true
         @in_fname = true
       when 'if', 'unless', 'while', 'until'
-        if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
+        if ((EXPR_MID | EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
           @lex_state = EXPR_BEG | EXPR_LABEL
         else
           @lex_state = EXPR_BEG
@@ -130,54 +132,54 @@ class RDoc::RipperStateLex
           @lex_state = EXPR_END
         end
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_beg(tok, data)
       @lex_state = EXPR_BEG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_end(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_CHAR(tok, data)
       @lex_state = EXPR_END
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_period(tok, data)
       @lex_state = EXPR_DOT
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_int(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_float(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rational(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_imaginary(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_symbeg(tok, data)
       @lex_state = EXPR_FNAME
       @continue = true
       @in_fname = true
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     private def on_variables(event, tok, data)
@@ -196,7 +198,7 @@ class RDoc::RipperStateLex
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
 
     def on_ident(tok, data)
@@ -225,32 +227,32 @@ class RDoc::RipperStateLex
 
     def on_lparen(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rparen(tok, data)
       @lex_state = EXPR_ENDFN
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbrace(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbrace(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbracket(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbracket(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_const(tok, data)
@@ -262,41 +264,43 @@ class RDoc::RipperStateLex
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_sp(tok, data)
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comma(tok, data)
       @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comment(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_sp(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+    end
+
+    def on_heredoc_beg(tok, data)
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+      @lex_state = EXPR_END
+      data
     end
 
     def on_heredoc_end(tok, data)
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
       @lex_state = EXPR_BEG
+      data
     end
 
     def on_default(event, tok, data)
       reset
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
   end unless RIPPER_HAS_LEX_STATE
 
@@ -306,21 +310,17 @@ class RDoc::RipperStateLex
     end
 
     def on_default(event, tok, data)
-      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => state})
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, state)
     end
   end if RIPPER_HAS_LEX_STATE
 
   def get_squashed_tk
     if @buf.empty?
-      tk = @inner_lex_enumerator.next
+      tk = @tokens.shift
     else
       tk = @buf.shift
     end
+    return nil if tk.nil?
     case tk[:kind]
     when :on_symbeg then
       tk = get_symbol_tk(tk)
@@ -367,7 +367,7 @@ class RDoc::RipperStateLex
 
   private def get_symbol_tk(tk)
     is_symbol = true
-    symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
+    symbol_tk = Token.new(tk.line_no, tk.char_no, :on_symbol)
     if ":'" == tk[:text] or ':"' == tk[:text]
       tk1 = get_string_tk(tk)
       symbol_tk[:text] = tk1[:text]
@@ -436,13 +436,7 @@ class RDoc::RipperStateLex
         end
       end
     end
-    {
-      :line_no => tk[:line_no],
-      :char_no => tk[:char_no],
-      :kind => kind,
-      :text => string,
-      :state => state
-    }
+    Token.new(tk.line_no, tk.char_no, kind, string, state)
   end
 
   private def get_regexp_tk(tk)
@@ -460,13 +454,7 @@ class RDoc::RipperStateLex
         string = string + inner_str_tk[:text]
       end
     end
-    {
-      :line_no => tk[:line_no],
-      :char_no => tk[:char_no],
-      :kind => :on_regexp,
-      :text => string,
-      :state => state
-    }
+    Token.new(tk.line_no, tk.char_no, :on_regexp, string, state)
   end
 
   private def get_embdoc_tk(tk)
@@ -475,20 +463,14 @@ class RDoc::RipperStateLex
       string = string + embdoc_tk[:text]
     end
     string = string + embdoc_tk[:text]
-    {
-      :line_no => tk[:line_no],
-      :char_no => tk[:char_no],
-      :kind => :on_embdoc,
-      :text => string,
-      :state => embdoc_tk[:state]
-    }
+    Token.new(tk.line_no, tk.char_no, :on_embdoc, string, embdoc_tk.state)
   end
 
   private def get_heredoc_tk(heredoc_name, indent)
     string = ''
     start_tk = nil
     prev_tk = nil
-    until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+    until heredoc_end?(heredoc_name, indent, tk = @tokens.shift) do
       start_tk = tk unless start_tk
       if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
         string = string + (' ' * tk[:char_no])
@@ -499,13 +481,7 @@ class RDoc::RipperStateLex
     start_tk = tk unless start_tk
     prev_tk = tk unless prev_tk
     @buf.unshift tk # closing heredoc
-    heredoc_tk = {
-      :line_no => start_tk[:line_no],
-      :char_no => start_tk[:char_no],
-      :kind => :on_heredoc,
-      :text => string,
-      :state => prev_tk[:state]
-    }
+    heredoc_tk = Token.new(start_tk.line_no, start_tk.char_no, :on_heredoc, string, prev_tk.state)
     @buf.unshift heredoc_tk
   end
 
@@ -561,13 +537,7 @@ class RDoc::RipperStateLex
       end
     end
     text = "#{start_token}#{string}#{end_token}"
-    {
-      :line_no => line_no,
-      :char_no => char_no,
-      :kind => :on_dstring,
-      :text => text,
-      :state => state
-    }
+    Token.new(line_no, char_no, :on_dstring, text, state)
   end
 
   private def get_op_tk(tk)
@@ -583,6 +553,10 @@ class RDoc::RipperStateLex
         tk[:text] += tk_ahead[:text]
         tk[:kind] = tk_ahead[:kind]
         tk[:state] = tk_ahead[:state]
+      when :on_heredoc_beg, :on_tstring, :on_dstring # frozen/non-frozen string literal
+        tk[:text] += tk_ahead[:text]
+        tk[:kind] = tk_ahead[:kind]
+        tk[:state] = tk_ahead[:state]
       else
         @buf.unshift tk_ahead
       end
@@ -594,11 +568,7 @@ class RDoc::RipperStateLex
     @buf = []
     @heredoc_queue = []
     @inner_lex = InnerStateLex.new(code)
-    @inner_lex_enumerator = Enumerator.new do |y|
-      @inner_lex.each do |tk|
-        y << tk
-      end
-    end
+    @tokens = @inner_lex.parse([])
   end
 
   def self.parse(code)