rdoc 7.2.0 → 8.1.0

data/lib/rdoc/text.rb

@@ -5,57 +5,30 @@
 
 require 'strscan'
 
-##
-# Methods for manipulating comment text
-
-module RDoc::Text
-
+module RDoc
   ##
-  # The language for this text.  This affects stripping comments
-  # markers.
+  # Methods for manipulating comment text
 
-  attr_accessor :language
+  module Text
 
-  ##
-  # Maps markup formats to classes that can parse them.  If the format is
-  # unknown, "rdoc" format is used.
+    ##
+    # The language for this text.  This affects stripping comments
+    # markers.
 
-  MARKUP_FORMAT = {
-    'markdown' => RDoc::Markdown,
-    'rdoc'     => RDoc::Markup,
-    'rd'       => RDoc::RD,
-    'tomdoc'   => RDoc::TomDoc,
-  }
+    attr_accessor :language
 
-  MARKUP_FORMAT.default = RDoc::Markup
+    ##
+    # Maps markup formats to classes that can parse them.  If the format is
+    # unknown, "rdoc" format is used.
 
-  ##
-  # Maps an encoding to a Hash of characters properly transcoded for that
-  # encoding.
-  #
-  # See also encode_fallback.
-
-  TO_HTML_CHARACTERS = Hash.new do |h, encoding|
-    h[encoding] = {
-      :close_dquote => encode_fallback('”', encoding, '"'),
-      :close_squote => encode_fallback('’', encoding, '\''),
-      :copyright    => encode_fallback('©', encoding, '(c)'),
-      :ellipsis     => encode_fallback('…', encoding, '...'),
-      :em_dash      => encode_fallback('—', encoding, '---'),
-      :en_dash      => encode_fallback('–', encoding, '--'),
-      :open_dquote  => encode_fallback('“', encoding, '"'),
-      :open_squote  => encode_fallback('‘', encoding, '\''),
-      :trademark    => encode_fallback('®', encoding, '(r)'),
+    MARKUP_FORMAT = {
+      'markdown' => Markdown,
+      'rdoc'     => Markup,
+      'rd'       => RD,
+      'tomdoc'   => TomDoc,
     }
-  end
-
-  ##
-  # Transcodes +character+ to +encoding+ with a +fallback+ character.
 
-  def self.encode_fallback(character, encoding, fallback)
-    character.encode(encoding, :fallback => { character => fallback },
-                     :undef => :replace, :replace => fallback)
-  end
+    MARKUP_FORMAT.default = Markup
 
   ##
   # Expands tab characters in +text+ to eight spaces
@@ -66,7 +39,7 @@ module RDoc::Text
     text.each_line do |line|
       nil while line.gsub!(/(?:\G|\r)((?:.{8})*?)([^\t\r\n]{0,7})\t/) do
         r = "#{$1}#{$2}#{' ' * (8 - $2.size)}"
-        r = RDoc::Encoding.change_encoding r, text.encoding
+        r = Encoding.change_encoding r, text.encoding
         r
       end
 
@@ -76,249 +49,160 @@ module RDoc::Text
     expanded.join
   end
 
-  ##
-  # Flush +text+ left based on the shortest line
+    ##
+    # Flush +text+ left based on the shortest line
 
-  def flush_left(text)
-    indent = 9999
+    def flush_left(text)
+      indent = 9999
 
-    text.each_line do |line|
-      line_indent = line =~ /\S/ || 9999
-      indent = line_indent if indent > line_indent
-    end
+      text.each_line do |line|
+        line_indent = line =~ /\S/ || 9999
+        indent = line_indent if indent > line_indent
+      end
 
-    empty = ''
-    empty = RDoc::Encoding.change_encoding empty, text.encoding
+      empty = ''
+      empty = Encoding.change_encoding empty, text.encoding
 
-    text.gsub(/^ {0,#{indent}}/, empty)
-  end
+      text.gsub(/^ {0,#{indent}}/, empty)
+    end
 
-  ##
-  # Convert a string in markup format into HTML.
-  #
-  # Requires the including class to implement #formatter
+    ##
+    # Convert a string in markup format into HTML.
+    #
+    # Requires the including class to implement #formatter
 
-  def markup(text)
-    if @store.options
-      locale = @store.options.locale
-    else
-      locale = nil
-    end
-    if locale
-      i18n_text = RDoc::I18n::Text.new(text)
-      text = i18n_text.translate(locale)
+    def markup(text)
+      if @store.options
+        locale = @store.options.locale
+      else
+        locale = nil
+      end
+      if locale
+        i18n_text = I18n::Text.new(text)
+        text = i18n_text.translate(locale)
+      end
+      parse(text).accept formatter
     end
-    parse(text).accept formatter
-  end
 
-  ##
-  # Strips hashes, expands tabs then flushes +text+ to the left
+    ##
+    # Strips hashes, expands tabs then flushes +text+ to the left
 
-  def normalize_comment(text)
-    return text if text.empty?
+    def normalize_comment(text)
+      return text if text.empty?
 
-    case language
-    when :ruby
-      text = strip_hashes text
-    when :c
-      text = strip_stars text
+      case language
+      when :ruby
+        text = strip_hashes text
+      when :c
+        text = strip_stars text
+      end
+      text = expand_tabs    text
+      text = flush_left     text
+      text = strip_newlines text
+      text
     end
-    text = expand_tabs    text
-    text = flush_left     text
-    text = strip_newlines text
-    text
-  end
-
-  ##
-  # Normalizes +text+ then builds a RDoc::Markup::Document from it
 
-  def parse(text, format = 'rdoc')
-    return text if RDoc::Markup::Document === text
-    return text.parse if RDoc::Comment === text
+    ##
+    # Normalizes +text+ then builds a RDoc::Markup::Document from it
 
-    text = normalize_comment text # TODO remove, should not be necessary
+    def parse(text, format = 'rdoc')
+      return text if Markup::Document === text
+      return text.parse if Comment === text
 
-    return RDoc::Markup::Document.new if text =~ /\A\n*\z/
+      text = normalize_comment text # TODO remove, should not be necessary
 
-    MARKUP_FORMAT[format].parse text
-  end
+      return Markup::Document.new if text =~ /\A\n*\z/
 
-  ##
-  # The first +limit+ characters of +text+ as HTML
+      MARKUP_FORMAT[format].parse text
+    end
 
-  def snippet(text, limit = 100)
-    document = parse text
+    ##
+    # The first +limit+ characters of +text+ as HTML
 
-    RDoc::Markup::ToHtmlSnippet.new(options, limit).convert document
-  end
+    def snippet(text, limit = 100)
+      document = parse text
 
-  ##
-  # Strips leading # characters from +text+
+      Markup::ToHtmlSnippet.new(limit).convert document
+    end
 
-  def strip_hashes(text)
-    return text if text =~ /^(?>\s*)[^\#]/
+    ##
+    # Strips leading # characters from +text+
 
-    empty = ''
-    empty = RDoc::Encoding.change_encoding empty, text.encoding
+    def strip_hashes(text)
+      return text if text =~ /^(?>\s*)[^\#]/
 
-    text.gsub(/^\s*(#+)/) { $1.tr '#', ' ' }.gsub(/^\s+$/, empty)
-  end
+      empty = ''
+      empty = Encoding.change_encoding empty, text.encoding
 
-  ##
-  # Strips leading and trailing \n characters from +text+
+      text.gsub(/^\s*(#+)/) { $1.tr '#', ' ' }.gsub(/^\s+$/, empty)
+    end
 
-  def strip_newlines(text)
-    text.gsub(/\A\n*(.*?)\n*\z/m) do $1 end # block preserves String encoding
-  end
+    ##
+    # Strips leading and trailing \n characters from +text+
 
-  ##
-  # Strips /* */ style comments
+    def strip_newlines(text)
+      text.gsub(/\A\n*(.*?)\n*\z/m) do $1 end # block preserves String encoding
+    end
 
-  def strip_stars(text)
-    return text unless text =~ %r%/\*.*\*/%m
+    ##
+    # Strips /* */ style comments
 
-    encoding = text.encoding
+    def strip_stars(text)
+      return text unless text =~ %r%/\*.*\*/%m
 
-    text = text.gsub %r%Document-method:\s+[\w:.#=!?|^&<>~+\-/*\%@`\[\]]+%, ''
+      encoding = text.encoding
 
-    space = ' '
-    space = RDoc::Encoding.change_encoding space, encoding if encoding
+      text = text.gsub %r%Document-method:\s+[\w:.#=!?|^&<>~+\-/*\%@`\[\]]+%, ''
 
-    text.sub!  %r%/\*+%       do space * $&.length end
-    text.sub!  %r%\*+/%       do space * $&.length end
-    text.gsub! %r%^[ \t]*\*%m do space * $&.length end
+      space = ' '
+      space = Encoding.change_encoding space, encoding if encoding
 
-    empty = ''
-    empty = RDoc::Encoding.change_encoding empty, encoding if encoding
-    text.gsub(/^\s+$/, empty)
-  end
+      text.sub!  %r%/\*+%       do space * $&.length end
+      text.sub!  %r%\*+/%       do space * $&.length end
+      text.gsub! %r%^[ \t]*\*%m do space * $&.length end
 
-  def to_html(text)
-    to_html_characters(text)
-  end
+      empty = ''
+      empty = Encoding.change_encoding empty, encoding if encoding
+      text.gsub(/^\s+$/, empty)
+    end
 
-  ##
-  # Converts ampersand, dashes, ellipsis, quotes, copyright and registered
-  # trademark symbols in +text+ to properly encoded characters.
-
-  def to_html_characters(text)
-    html = (''.encode text.encoding).dup
-
-    encoded = RDoc::Text::TO_HTML_CHARACTERS[text.encoding]
-
-    s = StringScanner.new text
-    insquotes = false
-    indquotes = false
-    after_word = nil
-
-    until s.eos? do
-      case
-      when s.scan(/<(tt|code)>.*?<\/\1>/) then # skip contents of tt
-        html << s.matched
-      when s.scan(/<(tt|code)>.*?/) then
-        warn "mismatched <#{s[1]}> tag" # TODO signal file/line
-        html << s.matched
-      when s.scan(/<[^>]+\/?s*>/) then # skip HTML tags
-        html << s.matched
-      when s.scan(/\.\.\.(\.?)/) then
-        html << s[1] << encoded[:ellipsis]
-        after_word = nil
-      when s.scan(/\(c\)/i) then
-        html << encoded[:copyright]
-        after_word = nil
-      when s.scan(/\(r\)/i) then
-        html << encoded[:trademark]
-        after_word = nil
-      when s.scan(/---/) then
-        html << encoded[:em_dash]
-        after_word = nil
-      when s.scan(/--/) then
-        html << encoded[:en_dash]
-        after_word = nil
-      when s.scan(/&quot;|"/) then
-        html << encoded[indquotes ? :close_dquote : :open_dquote]
-        indquotes = !indquotes
-        after_word = nil
-      when s.scan(/``/) then # backtick double quote
-        html << encoded[:open_dquote]
-        after_word = nil
-      when s.scan(/(?:&#39;|'){2}/) then # tick double quote
-        html << encoded[:close_dquote]
-        after_word = nil
-      when s.scan(/`/) then # backtick
-        if insquotes or after_word
-          html << '`'
-          after_word = false
-        else
-          html << encoded[:open_squote]
-          insquotes = true
-        end
-      when s.scan(/&#39;|'/) then # single quote
-        if insquotes
-          html << encoded[:close_squote]
-          insquotes = false
-        elsif after_word
-          # Mary's dog, my parents' house: do not start paired quotes
-          html << encoded[:close_squote]
-        else
-          html << encoded[:open_squote]
-          insquotes = true
-        end
+    ##
+    # Wraps +txt+ to +line_len+
 
-        after_word = nil
-      else # advance to the next potentially significant character
-        match = s.scan(/.+?(?=[<\\.("'`&-])/) #"
+    def wrap(txt, line_len = 76)
+      res = []
+      sp = 0
+      ep = txt.length
 
-        if match then
-          html << match
-          after_word = match =~ /\w$/
+      while sp < ep
+        # scan back for a space
+        p = sp + line_len - 1
+        if p >= ep
+          p = ep
         else
-          html << s.rest
-          break
-        end
-      end
-    end
-
-    html
-  end
-
-  ##
-  # Wraps +txt+ to +line_len+
-
-  def wrap(txt, line_len = 76)
-    res = []
-    sp = 0
-    ep = txt.length
-
-    while sp < ep
-      # scan back for a space
-      p = sp + line_len - 1
-      if p >= ep
-        p = ep
-      else
-        while p > sp and txt[p] != ?\s
-          p -= 1
-        end
-        if p <= sp
-          p = sp + line_len
-          while p < ep and txt[p] != ?\s
-            p += 1
+          while p > sp and txt[p] != ?\s
+            p -= 1
+          end
+          if p <= sp
+            p = sp + line_len
+            while p < ep and txt[p] != ?\s
+              p += 1
+            end
           end
         end
+        res << txt[sp...p] << "\n"
+        sp = p
+        sp += 1 while sp < ep and txt[sp] == ?\s
       end
-      res << txt[sp...p] << "\n"
-      sp = p
-      sp += 1 while sp < ep and txt[sp] == ?\s
-    end
 
-    res.join.strip
-  end
+      res.join.strip
+    end
 
-  ##
-  # Character class to be separated by a space when concatenating
-  # lines.
+    ##
+    # Character class to be separated by a space when concatenating
+    # lines.
 
-  SPACE_SEPARATED_LETTER_CLASS = /[\p{Nd}\p{Lc}\p{Pc}]|[!-~&&\W]/
+    SPACE_SEPARATED_LETTER_CLASS = /[\p{Nd}\p{Lc}\p{Pc}]|[!-~&&\W]/
 
   ##
   # Converts +text+ to a GitHub-style anchor ID:
@@ -358,4 +242,5 @@ module RDoc::Text
     label
   end
 
+  end
 end

data/lib/rdoc/token_stream.rb

@@ -1,126 +1,112 @@
 # frozen_string_literal: true
 
-##
-# A TokenStream is a list of tokens, gathered during the parse of some entity
-# (say a method). Entities populate these streams by being registered with the
-# lexer. Any class can collect tokens by including TokenStream. From the
-# outside, you use such an object by calling the start_collecting_tokens
-# method, followed by calls to add_token and pop_token.
-
-module RDoc::TokenStream
-
+module RDoc
   ##
-  # Converts +token_stream+ to HTML wrapping various tokens with
-  # <tt><span></tt> elements. Some tokens types are wrapped in spans
-  # with the given class names. Other token types are not wrapped in spans.
-
-  def self.to_html(token_stream)
-    starting_title = false
-
-    token_stream.map do |t|
-      next unless t
-
-      style = case t[:kind]
-              when :on_const   then 'ruby-constant'
-              when :on_kw      then 'ruby-keyword'
-              when :on_ivar    then 'ruby-ivar'
-              when :on_cvar    then 'ruby-identifier'
-              when :on_gvar    then 'ruby-identifier'
-              when '=' != t[:text] && :on_op
-                               then 'ruby-operator'
-              when :on_tlambda then 'ruby-operator'
-              when :on_ident   then 'ruby-identifier'
-              when :on_label   then 'ruby-value'
-              when :on_backref, :on_dstring
-                               then 'ruby-node'
-              when :on_comment then 'ruby-comment'
-              when :on_embdoc  then 'ruby-comment'
-              when :on_regexp  then 'ruby-regexp'
-              when :on_tstring then 'ruby-string'
-              when :on_int, :on_float,
-                   :on_rational, :on_imaginary,
-                   :on_heredoc,
-                   :on_symbol, :on_CHAR then 'ruby-value'
-              when :on_heredoc_beg, :on_heredoc_end
-                               then 'ruby-identifier'
-              end
-
-      text = t[:text]
-
-      if :on_ident == t[:kind] && starting_title
-        starting_title = false
-        style = 'ruby-identifier ruby-title'
-      end
-
-      if :on_kw == t[:kind] and 'def' == t[:text]
-        starting_title = true
+  # A TokenStream is a list of tokens, gathered during the parse of some entity
+  # (say a method). Entities populate these streams by being registered with the
+  # lexer. Any class can collect tokens by including TokenStream. From the
+  # outside, you use such an object by calling the start_collecting_tokens
+  # method, followed by calls to add_token and pop_token.
+
+  module TokenStream
+
+    ##
+    # Converts +token_stream+ to HTML wrapping various tokens with
+    # <tt><span></tt> elements. Some tokens types are wrapped in spans
+    # with the given class names. Other token types are not wrapped in spans.
+
+    def self.to_html(token_stream)
+      token_stream.map do |t|
+        next unless t
+
+        style = case t[:kind]
+                when :operator   then 'ruby-operator'
+                when :keyword    then 'ruby-keyword'
+                when :constant   then 'ruby-constant'
+                when :ivar       then 'ruby-ivar'
+                when :comment    then 'ruby-comment'
+                when :value      then 'ruby-value'
+                when :string     then 'ruby-string'
+                when :symbol     then 'ruby-value'
+                when :x_string   then 'ruby-string'
+                when :regexp     then 'ruby-regexp'
+                when :identifier then 'ruby-identifier'
+                end
+
+        text = t[:text]
+
+        text = CGI.escapeHTML text
+
+        if style
+          end_with_newline = text.end_with?("\n")
+          text = text.chomp if end_with_newline
+          "<span class=\"#{style}\">#{text}</span>#{"\n" if end_with_newline}"
+        else
+          text
+        end
+      end.join
+    end
+
+    ##
+    # Adds +tokens+ to the collected tokens
+
+    def add_tokens(tokens)
+      token_stream.concat(tokens)
+    end
+
+    ##
+    # Adds one +token+ to the collected tokens
+
+    def add_token(token)
+      token_stream.push(token)
+    end
+
+    ##
+    # Starts collecting tokens
+    #
+    # The optional +loader+ is called once on first access and its result is reused.
+
+    def collect_tokens(language, loader: nil)
+      @token_stream = []
+      @token_stream_loader = loader
+      @token_stream_language = language
+    end
+
+    alias start_collecting_tokens collect_tokens
+
+    ##
+    # Remove the last token from the collected tokens
+
+    def pop_token
+      token_stream.pop
+    end
+
+    ##
+    # Current token stream
+
+    def token_stream
+      if @token_stream_loader
+        @token_stream = @token_stream_loader.call
+        @token_stream_loader = nil
       end
+      @token_stream
+    end
 
-      text = CGI.escapeHTML text
-
-      if style then
-        end_with_newline = text.end_with?("\n")
-        text = text.chomp if end_with_newline
-        "<span class=\"#{style}\">#{text}</span>#{"\n" if end_with_newline}"
-      else
-        text
-      end
-    end.join
-  end
-
-  ##
-  # Adds +tokens+ to the collected tokens
-
-  def add_tokens(tokens)
-    @token_stream.concat(tokens)
-  end
-
-  ##
-  # Adds one +token+ to the collected tokens
-
-  def add_token(token)
-    @token_stream.push(token)
-  end
-
-  ##
-  # Starts collecting tokens
-  #
-
-  def collect_tokens(language)
-    @token_stream = []
-    @token_stream_language = language
-  end
-
-  alias start_collecting_tokens collect_tokens
-
-  ##
-  # Remove the last token from the collected tokens
-
-  def pop_token
-    @token_stream.pop
-  end
+    ##
+    # Returns a string representation of the token stream
 
-  ##
-  # Current token stream
-
-  def token_stream
-    @token_stream
-  end
-
-  ##
-  # Returns a string representation of the token stream
+    def tokens_to_s
+      (token_stream or return '').compact.map { |token| token[:text] }.join ''
+    end
 
-  def tokens_to_s
-    (token_stream or return '').compact.map { |token| token[:text] }.join ''
-  end
+    ##
+    # Returns the source language of the token stream as a string
+    #
+    # Returns 'c' or 'ruby'
 
-  ##
-  # Returns the source language of the token stream as a string
-  #
-  # Returns 'c' or 'ruby'
+    def source_language
+      @token_stream_language == :c ? 'c' : 'ruby'
+    end
 
-  def source_language
-    @token_stream_language == :c ? 'c' : 'ruby'
   end
-
 end