rdoc 6.3.4.1 → 8.0.0

data/lib/rdoc/parser/ruby_colorizer.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require 'prism'
+require 'set'
+
+# Ruby code syntax highlighter.
+# Colorize result is an array of +RDoc::Parser::RubyColorizer::ColoredToken+
+# Actual color for each token kind is determined elsewhere (e.g., HTML generator)
+module RDoc::Parser::RubyColorizer
+
+  ColoredToken = Struct.new(:kind, :text)
+
+  # Prism operator token types except assignment '='
+  OP_TOKENS = %i[
+    AMPERSAND AMPERSAND_AMPERSAND
+    BANG BANG_EQUAL BANG_TILDE CARET COLON COLON_COLON
+    EQUAL_EQUAL EQUAL_GREATER EQUAL_TILDE
+    GREATER GREATER_GREATER
+    LESS LESS_EQUAL LESS_EQUAL_GREATER LESS_LESS
+    MINUS MINUS_GREATER PERCENT PIPE PIPE_PIPE PLUS
+    QUESTION_MARK SLASH STAR STAR_STAR TILDE
+    UAMPERSAND UMINUS UPLUS USTAR USTAR_STAR
+  ].to_set
+
+  # Prism token type to ColoredToken kind map
+  TOKEN_TYPE_MAP = {
+    IDENTIFIER: :identifier,
+    METHOD_NAME: :identifier,
+    INSTANCE_VARIABLE: :ivar,
+    CLASS_VARIABLE: :identifier,
+    GLOBAL_VARIABLE: :identifier,
+    BACK_REFERENCE: :identifier,
+    NUMBERED_REFERENCE: :identifier,
+    CONSTANT: :constant,
+    LABEL: :value,
+    INTEGER: :value,
+    INTEGER_IMAGINARY: :value,
+    INTEGER_RATIONAL: :value,
+    INTEGER_RATIONAL_IMAGINARY: :value,
+    FLOAT: :value,
+    FLOAT_IMAGINARY: :value,
+    FLOAT_RATIONAL: :value,
+    FLOAT_RATIONAL_IMAGINARY: :value,
+    COMMENT: :comment,
+    EMBDOC_BEGIN: :comment,
+    EMBDOC_LINE: :comment,
+    EMBDOC_END: :comment
+  }
+
+  class << self
+
+    # Colorize the entire +code+ and returns colored token stream.
+    def colorize(code)
+      result = Prism.parse_lex(code)
+      program_node, unordered_tokens = result.value
+      prism_tokens = unordered_tokens.map(&:first).sort_by! { |token| token.location.start_offset }
+      partial_colorize(code, program_node, prism_tokens, 0, code.bytesize)
+    end
+
+    # Colorize partial +node+ in +whole_code+ and returns colored token stream.
+    def partial_colorize(whole_code, node, prism_tokens, start_offset = nil, end_offset = nil)
+      start_offset ||= node.location.start_offset
+      end_offset ||= node.location.end_offset
+      visitor = NodeColorizeVisitor.new
+      node.accept(visitor)
+      prior_tokens = visitor.tokens.sort_by {|_, start_offset, _| start_offset }
+      normal_tokens = normal_tokens(slice_by_location(prism_tokens, start_offset, end_offset))
+      colored_tokens = unify_tokens(whole_code, prior_tokens, normal_tokens, start_offset, end_offset)
+      colored_tokens.unshift(ColoredToken.new(:plain, ' ' * node.location.start_column)) if node.location.start_column > 0
+      colored_tokens
+    end
+
+    private
+
+    def slice_by_location(items, start_offset, end_offset)
+      start_index = items.bsearch_index { |item| item.location.end_offset > start_offset } || items.size
+      end_index = items.bsearch_index { |item| item.location.start_offset >= end_offset } || items.size
+      items[start_index...end_index]
+    end
+
+    # Unify prior tokens and normal tokens into a single token stream.
+    # Prior tokens have higher priority than normal tokens.
+    # Also adds missing text (spaces, newlines, etc.) as :plain tokens
+    # so that the entire range is covered.
+    def unify_tokens(whole_code, prior_tokens, normal_tokens, start_offset, end_offset)
+      tokens = []
+      offset = start_offset
+
+      # Add missing text such as spaces and newlines as a separate :plain token
+      flush = -> next_offset {
+        return if offset == next_offset
+
+        whole_code.byteslice(offset...next_offset).scan(/\n|\s+|[^\s]+/) do |text|
+          tokens << ColoredToken.new(:plain, text)
+        end
+      }
+
+      until prior_tokens.empty? && normal_tokens.empty?
+        ptok = prior_tokens.first
+        ntok = normal_tokens.first
+        if ntok && (!ptok || ntok[2] <= ptok[1])
+          token = normal_tokens.shift
+        else
+          token = prior_tokens.shift
+        end
+        kind, start_pos, end_pos = token
+        next if start_pos < offset
+
+        flush.call(start_pos)
+        tokens << ColoredToken.new(kind, whole_code.byteslice(start_pos...end_pos))
+        offset = end_pos
+      end
+      flush.call(end_offset)
+      tokens
+    end
+
+    # Convert normal Prism tokens to [kind, start_offset, end_offset]
+    def normal_tokens(tokens)
+      tokens.map do |token,|
+        kind =
+          if token.type.start_with?('KEYWORD_')
+            :keyword
+          elsif OP_TOKENS.include?(token.type.to_sym)
+            :operator
+          else
+            TOKEN_TYPE_MAP[token.type] || :plain
+          end
+        [kind, token.location.start_offset, token.location.end_offset]
+      end
+    end
+  end
+
+  # Visitor to determine node colorizing which can't be determined by tokens.
+  # STRING_CONTENT/EMBEXPR_BEGIN/EMBEXPR_END in string/regexp/symbol have different colorizing
+  class NodeColorizeVisitor < Prism::Visitor # :nodoc:
+    attr_reader :tokens
+
+    def initialize
+      @tokens = []
+    end
+
+    def visit_symbol_node(node)
+      # SymbolNode#location may contain heredoc content and closing
+      # e.g., `<<A; :\\\nA\nsymbol`
+      # So we need to colorize opening, content and closing separately.
+      push_location(:symbol, node.opening_loc)
+      push_location(:symbol, node.value_loc)
+      push_location(:symbol, node.closing_loc)
+    end
+
+    def visit_interpolated_symbol_node(node)
+      push_location(:symbol, node.opening_loc)
+      handle_interpolated_parts(:symbol, node.parts)
+      push_location(:symbol, node.closing_loc)
+    end
+
+    def visit_regular_expression_node(node)
+      push_location(:regexp, node.location)
+    end
+
+    def visit_interpolated_regular_expression_node(node)
+      push_location(:regexp, node.opening_loc)
+      handle_interpolated_parts(:regexp, node.parts)
+      push_location(:regexp, node.closing_loc)
+    end
+
+    alias visit_match_last_line_node visit_regular_expression_node
+    alias visit_interpolated_match_last_line_node visit_interpolated_regular_expression_node
+
+    def visit_string_node(node)
+      # Node's location may not cover the entire string literal.
+      # For example, in a heredoc string, the node's location covers only the heredoc opening.
+      # We need to colorize opening, content and closing separately.
+      push_location(:string, node.opening_loc)
+      push_location(:string, node.content_loc)
+      push_location(:string, node.closing_loc)
+    end
+
+    def visit_interpolated_string_node(node)
+      push_location(:string, node.opening_loc)
+      handle_interpolated_parts(:string, node.parts)
+      push_location(:string, node.closing_loc)
+    end
+
+    def visit_x_string_node(node)
+      # Same as visit_string_node, node.location of <<`X` only covers opening,
+      # so we need to colorize opening, content and closing separately.
+      push_location(:x_string, node.opening_loc)
+      push_location(:x_string, node.content_loc)
+      push_location(:x_string, node.closing_loc)
+    end
+
+    def visit_interpolated_x_string_node(node)
+      push_location(:x_string, node.opening_loc)
+      handle_interpolated_parts(:x_string, node.parts)
+      push_location(:x_string, node.closing_loc)
+    end
+
+    def visit_array_node(node)
+      super
+      # Colorize %w[...] array literal like string literals, and %i[...] like symbol literals
+      case node.opening
+      when /\A%[wW].\z/
+        push_location(:string, node.opening_loc)
+        push_location(:string, node.closing_loc)
+      when /\A%[iI].\z/
+        push_location(:symbol, node.opening_loc)
+        push_location(:symbol, node.closing_loc)
+      end
+    end
+
+    def visit_def_node(node)
+      # For special colorizing of method name in def node
+      # e.g., `def <=>; end`
+      push_location(:identifier, node.name_loc)
+      super
+    end
+
+    private
+
+    def push_location(kind, location)
+      # Only push tokens that have a non-zero length
+      if location && location.start_offset < location.end_offset
+        @tokens << [kind, location.start_offset, location.end_offset]
+      end
+    end
+
+    def handle_interpolated_parts(kind, parts)
+      # StringNode, EmbeddedStatementsNode brackets, and EmbeddedVariableNode hash in
+      # interpolated regexp/symbol/string parts should be colored as regexp/symbol/string respectively.
+      parts.each do |part|
+        case part
+        when Prism::StringNode
+          # InterpolatedStringNode#parts may have its own opening/closing. e.g., `'a' "b"`
+          push_location(kind, part.opening_loc)
+          push_location(kind, part.content_loc)
+          push_location(kind, part.closing_loc)
+        when Prism::InterpolatedStringNode
+          # InterpolatedStringNode#parts may contain InterpolatedStringNode. e.g., `'a' "#{}"`
+          part.accept(self)
+        when Prism::EmbeddedStatementsNode
+          push_location(kind, part.opening_loc)
+          push_location(kind, part.closing_loc)
+          part.accept(self)
+        when Prism::EmbeddedVariableNode
+          push_location(kind, part.operator_loc)
+        end
+      end
+    end
+  end
+
+  private_constant :NodeColorizeVisitor
+end

data/lib/rdoc/parser/simple.rb

@@ -14,22 +14,22 @@ class RDoc::Parser::Simple < RDoc::Parser
   ##
   # Prepare to parse a plain file
 
-  def initialize(top_level, file_name, content, options, stats)
+  def initialize(top_level, content, options, stats)
     super
 
     preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
 
-    @content = preprocess.handle @content, @top_level
+    content = RDoc::Text.expand_tabs(@content)
+    @content, = preprocess.run_pre_processes(content, @top_level, 1, :simple)
   end
 
   ##
   # Extract the file contents and attach them to the TopLevel as a comment
 
   def scan
-    comment = remove_coding_comment @content
-    comment = remove_private_comment comment
+    content = remove_coding_comment @content
 
-    comment = RDoc::Comment.new comment, @top_level
+    comment = RDoc::Comment.new content, @top_level
 
     @top_level.comment = comment
     @top_level
@@ -38,24 +38,7 @@ class RDoc::Parser::Simple < RDoc::Parser
   ##
   # Removes the encoding magic comment from +text+
 
-  def remove_coding_comment text
+  def remove_coding_comment(text)
     text.sub(/\A# .*coding[=:].*$/, '')
   end
-
-  ##
-  # Removes private comments.
-  #
-  # Unlike RDoc::Comment#remove_private this implementation only looks for two
-  # dashes at the beginning of the line.  Three or more dashes are considered
-  # to be a rule and ignored.
-
-  def remove_private_comment comment
-    # Workaround for gsub encoding for Ruby 1.9.2 and earlier
-    empty = ''
-    empty = RDoc::Encoding.change_encoding empty, comment.encoding
-
-    comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
-    comment.sub(%r%^--\n.*%m, empty)
-  end
-
 end

data/lib/rdoc/parser/text.rb

@@ -9,4 +9,3 @@
 
 module RDoc::Parser::Text
 end
-

data/lib/rdoc/parser.rb

@@ -91,7 +91,7 @@ class RDoc::Parser
   # Checks if +file+ is a zip file in disguise.  Signatures from
   # http://www.garykessler.net/library/file_sigs.html
 
-  def self.zip? file
+  def self.zip?(file)
     zip_signature = File.read file, 4
 
     zip_signature == "PK\x03\x04" or
@@ -104,7 +104,7 @@ class RDoc::Parser
   ##
   # Return a parser that can handle a particular extension
 
-  def self.can_parse file_name
+  def self.can_parse(file_name)
     parser = can_parse_by_name file_name
 
     # HACK Selenium hides a jar file using a .txt extension
@@ -117,7 +117,7 @@ class RDoc::Parser
   # Returns a parser that can handle the extension for +file_name+.  This does
   # not depend upon the file being readable.
 
-  def self.can_parse_by_name file_name
+  def self.can_parse_by_name(file_name)
     _, parser = RDoc::Parser.parsers.find { |regexp,| regexp =~ file_name }
 
     # The default parser must not parse binary files
@@ -125,9 +125,11 @@ class RDoc::Parser
     return parser if ext_name.empty?
 
     if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/ then
-      case check_modeline file_name
+      case mode = check_modeline(file_name)
       when nil, 'rdoc' then # continue
-      else return nil
+      else
+        RDoc::Parser.parsers.find { |_, p| return p if mode.casecmp?(p.name[/\w+\z/]) }
+        return nil
       end
     end
 
@@ -138,7 +140,7 @@ class RDoc::Parser
   ##
   # Returns the file type from the modeline in +file_name+
 
-  def self.check_modeline file_name
+  def self.check_modeline(file_name)
     line = File.open file_name do |io|
       io.gets
     end
@@ -164,7 +166,8 @@ class RDoc::Parser
   # Finds and instantiates the correct parser for the given +file_name+ and
   # +content+.
 
-  def self.for top_level, file_name, content, options, stats
+  def self.for(top_level, content, options, stats)
+    file_name = top_level.absolute_name
     return if binary? file_name
 
     parser = use_markup content
@@ -188,7 +191,7 @@ class RDoc::Parser
 
     content = remove_modeline content
 
-    parser.new top_level, file_name, content, options, stats
+    parser.new top_level, content, options, stats
   rescue SystemCallError
     nil
   end
@@ -205,7 +208,7 @@ class RDoc::Parser
   ##
   # Removes an emacs-style modeline from the first line of the document
 
-  def self.remove_modeline content
+  def self.remove_modeline(content)
     content.sub(/\A.*-\*-\s*(.*?\S)\s*-\*-.*\r?\n/, '')
   end
 
@@ -226,7 +229,7 @@ class RDoc::Parser
   #
   # Any comment style may be used to hide the markup comment.
 
-  def self.use_markup content
+  def self.use_markup(content)
     markup = content.lines.first(3).grep(/markup:\s+(\w+)/) { $1 }.first
 
     return unless markup
@@ -249,12 +252,12 @@ class RDoc::Parser
   # RDoc::Markup::PreProcess object is created which allows processing of
   # directives.
 
-  def initialize top_level, file_name, content, options, stats
+  def initialize(top_level, content, options, stats)
     @top_level = top_level
     @top_level.parser = self.class
     @store = @top_level.store
 
-    @file_name = file_name
+    @file_name = top_level.absolute_name
     @content = content
     @options = options
     @stats = stats
@@ -263,9 +266,25 @@ class RDoc::Parser
     @preprocess.options = @options
   end
 
-  autoload :RubyTools, 'rdoc/parser/ruby_tools'
-  autoload :Text,      'rdoc/parser/text'
+  autoload :Text, "#{__dir__}/parser/text"
 
+  ##
+  # Normalizes tabs in +body+
+
+  def handle_tab_width(body)
+    if /\t/ =~ body
+      tab_width = @options.tab_width
+      body.split(/\n/).map do |line|
+        1 while line.gsub!(/\t+/) do
+          b, e = $~.offset(0)
+          ' ' * (tab_width * (e-b) - b % tab_width)
+        end
+        line
+      end.join "\n"
+    else
+      body
+    end
+  end
 end
 
 # simple must come first in order to show up last in the parsers list
@@ -274,4 +293,6 @@ require_relative 'parser/c'
 require_relative 'parser/changelog'
 require_relative 'parser/markdown'
 require_relative 'parser/rd'
+require_relative 'parser/rbs'
 require_relative 'parser/ruby'
+require_relative 'parser/ruby_colorizer'

data/lib/rdoc/rbs_helper.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'pathname'
+require 'rbs'
+require 'rdoc/markup/formatter'
+
+##
+# RBS type signature support.
+# Loads type information from .rbs files, validates inline annotations,
+# and converts type signatures to HTML with linked type names.
+
+module RDoc
+  module RbsHelper
+    class << self
+
+      ##
+      # Returns true if +sig+ is a valid RBS method type signature.
+
+      def valid_method_type?(sig)
+        RBS::Parser.parse_method_type(sig, require_eof: true)
+        true
+      rescue RBS::ParsingError
+        false
+      end
+
+      ##
+      # Returns true if +sig+ is a valid RBS type signature.
+
+      def valid_type?(sig)
+        RBS::Parser.parse_type(sig, require_eof: true)
+        true
+      rescue RBS::ParsingError
+        false
+      end
+
+      ##
+      # Loads RBS signatures from the given directories.
+      # Returns a Hash mapping "ClassName#method_name" => ["type sig string", ...].
+
+      def load_signatures(*dirs)
+        loader = RBS::EnvironmentLoader.new
+        dirs.each { |dir| loader.add(path: Pathname(dir)) }
+
+        env = RBS::Environment.new
+        loader.load(env: env)
+
+        signatures = {}
+
+        env.class_decls.each do |type_name, entry|
+          class_name = type_name.to_s.delete_prefix('::')
+
+          entry.each_decl do |decl|
+            decl.members.each do |member|
+              case member
+              when RBS::AST::Members::MethodDefinition
+                sigs = member.overloads.map { |o| o.method_type.to_s }
+                method_keys_for(class_name, member).each do |key|
+                  signatures[key] ||= sigs
+                end
+              when RBS::AST::Members::AttrReader, RBS::AST::Members::AttrWriter, RBS::AST::Members::AttrAccessor
+                key = member.kind == :singleton ? "#{class_name}.#{member.name}" : "#{class_name}##{member.name}"
+                signatures[key] ||= [member.type.to_s]
+              end
+            end
+          end
+        end
+
+        signatures
+      end
+
+      ##
+      # Converts type signature lines to HTML with type names linked to
+      # their documentation pages. Uses the RBS parser to extract type
+      # name locations precisely.
+      #
+      # +lines+ is an Array of signature line strings.
+      # +lookup+ is a Hash mapping type names to their doc paths.
+      # +from_path+ is the current page path for generating relative URLs.
+      #
+      # Returns escaped HTML with +->+ replaced by +→+.
+
+      def signature_to_html(lines, lookup:, from_path:)
+        lines.map { |line|
+          link_type_names_in_line(line, lookup, from_path).gsub('-&gt;', '&rarr;')
+        }.join("\n")
+      end
+
+      private
+
+      # `def self?.foo: ...` produces a member whose kind is :singleton_instance —
+      # it defines both Class.foo (singleton) and a private Class#foo (instance),
+      # so we need to register the signature under both keys.
+      def method_keys_for(class_name, member)
+        case member.kind
+        when :singleton
+          ["#{class_name}.#{member.name}"]
+        when :singleton_instance
+          ["#{class_name}.#{member.name}", "#{class_name}##{member.name}"]
+        else
+          ["#{class_name}##{member.name}"]
+        end
+      end
+
+      def link_type_names_in_line(line, lookup, from_path)
+        escaped = ERB::Util.html_escape(line)
+
+        locs = collect_type_name_locations(line)
+        return escaped if locs.empty?
+
+        result = escaped.dup
+
+        # Replace type names with links, working backwards to preserve positions.
+        # HTML escaping (e.g. -> becomes -&gt;) shifts positions, so we
+        # re-escape the prefix to find the correct offset in the result.
+        locs.sort_by { |l| -l[:start] }.each do |loc|
+          name = loc[:name]
+          next unless (target_path = lookup[name])
+
+          prefix = ERB::Util.html_escape(line[0...loc[:start]])
+          escaped_name = ERB::Util.html_escape(name)
+          start_in_escaped = prefix.length
+          end_in_escaped = start_in_escaped + escaped_name.length
+
+          href = ERB::Util.html_escape(::RDoc::Markup::Formatter.gen_relative_url(from_path, target_path))
+          result[start_in_escaped...end_in_escaped] =
+            "<a href=\"#{href}\" class=\"rbs-type\">#{escaped_name}</a>"
+        end
+
+        result
+      end
+
+      ##
+      # Extracts type name locations from a signature line using the RBS parser.
+
+      def collect_type_name_locations(line)
+        locs = []
+
+        begin
+          mt = RBS::Parser.parse_method_type(line, require_eof: true)
+        rescue RBS::ParsingError
+          begin
+            type = RBS::Parser.parse_type(line, require_eof: true)
+            collect_from_type(type, locs)
+            return locs
+          rescue RBS::ParsingError
+            return locs
+          end
+        end
+
+        mt.type.each_param { |p| collect_from_type(p.type, locs) }
+        if mt.block
+          mt.block.type.each_param { |p| collect_from_type(p.type, locs) }
+          collect_from_type(mt.block.type.return_type, locs)
+        end
+        collect_from_type(mt.type.return_type, locs)
+
+        locs
+      end
+
+      ##
+      # Recursively collects type name locations from an RBS type AST node.
+
+      def collect_from_type(type, locs)
+        case type
+        when RBS::Types::ClassInstance
+          name = type.name.to_s.delete_prefix('::')
+          if type.location
+            name_loc = type.location[:name] || type.location
+            locs << { name: name, start: name_loc.end_pos - name.length }
+          end
+          type.args.each { |a| collect_from_type(a, locs) }
+        when RBS::Types::Union, RBS::Types::Intersection, RBS::Types::Tuple
+          type.types.each { |t| collect_from_type(t, locs) }
+        when RBS::Types::Optional
+          collect_from_type(type.type, locs)
+        when RBS::Types::Record
+          type.all_fields.each_value { |t| collect_from_type(t, locs) }
+        when RBS::Types::Proc
+          type.type.each_param { |p| collect_from_type(p.type, locs) }
+          collect_from_type(type.type.return_type, locs)
+        end
+      end
+    end
+  end
+end