rdoc 7.2.0 → 8.1.0

data/lib/rdoc/parser/ruby_colorizer.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require 'prism'
+require 'set'
+
+module RDoc
+  class Parser
+    # Ruby code syntax highlighter.
+    # Colorize result is an array of +RDoc::Parser::RubyColorizer::ColoredToken+
+    # Actual color for each token kind is determined elsewhere (e.g., HTML generator)
+    module RubyColorizer
+
+      ColoredToken = Struct.new(:kind, :text)
+
+      # Defers colorization for all nodes in one source file until first access.
+      class DeferredContext # :nodoc:
+        #: (String) -> void
+        def initialize(source)
+          @source = source
+          @token_streams = {}
+        end
+
+        #: (Integer) -> ^() -> Array[ColoredToken]
+        def token_stream_loader(node_id)
+          @token_streams[node_id] = nil
+          -> { token_stream_for(node_id) }
+        end
+
+        #: (Integer) -> Array[ColoredToken]
+        def token_stream_for(node_id)
+          materialize unless materialized?
+          @token_streams.fetch(node_id)
+        end
+
+        private
+
+        #: () -> Boolean
+        def materialized?
+          !@source
+        end
+
+        #: () -> void
+        def materialize
+          program_node, unordered_tokens = Prism.parse_lex(@source).value
+          prism_tokens = unordered_tokens.map(&:first).sort_by! { |token| token.location.start_offset }
+          staged_tokens = {}
+          nodes = [program_node]
+          until nodes.empty?
+            node = nodes.pop
+            if @token_streams.key?(node.node_id)
+              staged_tokens[node.node_id] = RubyColorizer.partial_colorize(@source, node, prism_tokens)
+            end
+            nodes.concat(node.compact_child_nodes)
+          end
+
+          @token_streams = staged_tokens
+          @source = nil
+        end
+      end
+
+      # Prism operator token types except assignment '='
+      OP_TOKENS = %i[
+        AMPERSAND AMPERSAND_AMPERSAND
+        BANG BANG_EQUAL BANG_TILDE CARET COLON COLON_COLON
+        EQUAL_EQUAL EQUAL_GREATER EQUAL_TILDE
+        GREATER GREATER_GREATER
+        LESS LESS_EQUAL LESS_EQUAL_GREATER LESS_LESS
+        MINUS MINUS_GREATER PERCENT PIPE PIPE_PIPE PLUS
+        QUESTION_MARK SLASH STAR STAR_STAR TILDE
+        UAMPERSAND UMINUS UPLUS USTAR USTAR_STAR
+      ].to_set
+
+      # Prism token type to ColoredToken kind map
+      TOKEN_TYPE_MAP = {
+        IDENTIFIER: :identifier,
+        METHOD_NAME: :identifier,
+        INSTANCE_VARIABLE: :ivar,
+        CLASS_VARIABLE: :identifier,
+        GLOBAL_VARIABLE: :identifier,
+        BACK_REFERENCE: :identifier,
+        NUMBERED_REFERENCE: :identifier,
+        CONSTANT: :constant,
+        LABEL: :value,
+        INTEGER: :value,
+        INTEGER_IMAGINARY: :value,
+        INTEGER_RATIONAL: :value,
+        INTEGER_RATIONAL_IMAGINARY: :value,
+        FLOAT: :value,
+        FLOAT_IMAGINARY: :value,
+        FLOAT_RATIONAL: :value,
+        FLOAT_RATIONAL_IMAGINARY: :value,
+        COMMENT: :comment,
+        EMBDOC_BEGIN: :comment,
+        EMBDOC_LINE: :comment,
+        EMBDOC_END: :comment
+      }
+
+      class << self
+
+        # Colorize the entire +code+ and returns colored token stream.
+        def colorize(code)
+          result = Prism.parse_lex(code)
+          program_node, unordered_tokens = result.value
+          prism_tokens = unordered_tokens.map(&:first).sort_by! { |token| token.location.start_offset }
+          partial_colorize(code, program_node, prism_tokens, 0, code.bytesize)
+        end
+
+        # Colorize partial +node+ in +whole_code+ and returns colored token stream.
+        def partial_colorize(whole_code, node, prism_tokens, start_offset = nil, end_offset = nil)
+          start_offset ||= node.location.start_offset
+          end_offset ||= node.location.end_offset
+          visitor = NodeColorizeVisitor.new
+          node.accept(visitor)
+          prior_tokens = visitor.tokens.sort_by {|_, start_offset, _| start_offset }
+          normal_tokens = normal_tokens(slice_by_location(prism_tokens, start_offset, end_offset))
+          colored_tokens = unify_tokens(whole_code, prior_tokens, normal_tokens, start_offset, end_offset)
+          colored_tokens.unshift(ColoredToken.new(:plain, ' ' * node.location.start_column)) if node.location.start_column > 0
+          colored_tokens
+        end
+
+        private
+
+        def slice_by_location(items, start_offset, end_offset)
+          start_index = items.bsearch_index { |item| item.location.end_offset > start_offset } || items.size
+          end_index = items.bsearch_index { |item| item.location.start_offset >= end_offset } || items.size
+          items[start_index...end_index]
+        end
+
+        # Unify prior tokens and normal tokens into a single token stream.
+        # Prior tokens have higher priority than normal tokens.
+        # Also adds missing text (spaces, newlines, etc.) as :plain tokens
+        # so that the entire range is covered.
+        def unify_tokens(whole_code, prior_tokens, normal_tokens, start_offset, end_offset)
+          tokens = []
+          offset = start_offset
+
+          # Add missing text such as spaces and newlines as a separate :plain token
+          flush = -> next_offset {
+            return if offset == next_offset
+
+            whole_code.byteslice(offset...next_offset).scan(/\n|\s+|[^\s]+/) do |text|
+              tokens << ColoredToken.new(:plain, text)
+            end
+          }
+
+          until prior_tokens.empty? && normal_tokens.empty?
+            ptok = prior_tokens.first
+            ntok = normal_tokens.first
+            if ntok && (!ptok || ntok[2] <= ptok[1])
+              token = normal_tokens.shift
+            else
+              token = prior_tokens.shift
+            end
+            kind, start_pos, end_pos = token
+            next if start_pos < offset
+
+            flush.call(start_pos)
+            tokens << ColoredToken.new(kind, whole_code.byteslice(start_pos...end_pos))
+            offset = end_pos
+          end
+          flush.call(end_offset)
+          tokens
+        end
+
+        # Convert normal Prism tokens to [kind, start_offset, end_offset]
+        def normal_tokens(tokens)
+          tokens.map do |token,|
+            kind =
+              if token.type.start_with?('KEYWORD_')
+                :keyword
+              elsif OP_TOKENS.include?(token.type.to_sym)
+                :operator
+              else
+                TOKEN_TYPE_MAP[token.type] || :plain
+              end
+            [kind, token.location.start_offset, token.location.end_offset]
+          end
+        end
+      end
+
+      # Visitor to determine node colorizing which can't be determined by tokens.
+      # STRING_CONTENT/EMBEXPR_BEGIN/EMBEXPR_END in string/regexp/symbol have different colorizing
+      class NodeColorizeVisitor < Prism::Visitor # :nodoc:
+        attr_reader :tokens
+
+        def initialize
+          @tokens = []
+        end
+
+        def visit_symbol_node(node)
+          # SymbolNode#location may contain heredoc content and closing
+          # e.g., `<<A; :\\\nA\nsymbol`
+          # So we need to colorize opening, content and closing separately.
+          push_location(:symbol, node.opening_loc)
+          push_location(:symbol, node.value_loc)
+          push_location(:symbol, node.closing_loc)
+        end
+
+        def visit_interpolated_symbol_node(node)
+          push_location(:symbol, node.opening_loc)
+          handle_interpolated_parts(:symbol, node.parts)
+          push_location(:symbol, node.closing_loc)
+        end
+
+        def visit_regular_expression_node(node)
+          push_location(:regexp, node.location)
+        end
+
+        def visit_interpolated_regular_expression_node(node)
+          push_location(:regexp, node.opening_loc)
+          handle_interpolated_parts(:regexp, node.parts)
+          push_location(:regexp, node.closing_loc)
+        end
+
+        alias visit_match_last_line_node visit_regular_expression_node
+        alias visit_interpolated_match_last_line_node visit_interpolated_regular_expression_node
+
+        def visit_string_node(node)
+          # Node's location may not cover the entire string literal.
+          # For example, in a heredoc string, the node's location covers only the heredoc opening.
+          # We need to colorize opening, content and closing separately.
+          push_location(:string, node.opening_loc)
+          push_location(:string, node.content_loc)
+          push_location(:string, node.closing_loc)
+        end
+
+        def visit_interpolated_string_node(node)
+          push_location(:string, node.opening_loc)
+          handle_interpolated_parts(:string, node.parts)
+          push_location(:string, node.closing_loc)
+        end
+
+        def visit_x_string_node(node)
+          # Same as visit_string_node, node.location of <<`X` only covers opening,
+          # so we need to colorize opening, content and closing separately.
+          push_location(:x_string, node.opening_loc)
+          push_location(:x_string, node.content_loc)
+          push_location(:x_string, node.closing_loc)
+        end
+
+        def visit_interpolated_x_string_node(node)
+          push_location(:x_string, node.opening_loc)
+          handle_interpolated_parts(:x_string, node.parts)
+          push_location(:x_string, node.closing_loc)
+        end
+
+        def visit_array_node(node)
+          super
+          # Colorize %w[...] array literal like string literals, and %i[...] like symbol literals
+          case node.opening
+          when /\A%[wW].\z/
+            push_location(:string, node.opening_loc)
+            push_location(:string, node.closing_loc)
+          when /\A%[iI].\z/
+            push_location(:symbol, node.opening_loc)
+            push_location(:symbol, node.closing_loc)
+          end
+        end
+
+        def visit_def_node(node)
+          # For special colorizing of method name in def node
+          # e.g., `def <=>; end`
+          push_location(:identifier, node.name_loc)
+          super
+        end
+
+        private
+
+        def push_location(kind, location)
+          # Only push tokens that have a non-zero length
+          if location && location.start_offset < location.end_offset
+            @tokens << [kind, location.start_offset, location.end_offset]
+          end
+        end
+
+        def handle_interpolated_parts(kind, parts)
+          # StringNode, EmbeddedStatementsNode brackets, and EmbeddedVariableNode hash in
+          # interpolated regexp/symbol/string parts should be colored as regexp/symbol/string respectively.
+          parts.each do |part|
+            case part
+            when Prism::StringNode
+              # InterpolatedStringNode#parts may have its own opening/closing. e.g., `'a' "b"`
+              push_location(kind, part.opening_loc)
+              push_location(kind, part.content_loc)
+              push_location(kind, part.closing_loc)
+            when Prism::InterpolatedStringNode
+              # InterpolatedStringNode#parts may contain InterpolatedStringNode. e.g., `'a' "#{}"`
+              part.accept(self)
+            when Prism::EmbeddedStatementsNode
+              push_location(kind, part.opening_loc)
+              push_location(kind, part.closing_loc)
+              part.accept(self)
+            when Prism::EmbeddedVariableNode
+              push_location(kind, part.operator_loc)
+            end
+          end
+        end
+      end
+
+      private_constant :NodeColorizeVisitor
+    end
+  end
+end

data/lib/rdoc/parser/simple.rb

@@ -1,44 +1,48 @@
 # frozen_string_literal: true
-##
-# Parse a non-source file. We basically take the whole thing as one big
-# comment.
+module RDoc
+  class Parser
+    ##
+    # Parse a non-source file. We basically take the whole thing as one big
+    # comment.
 
-class RDoc::Parser::Simple < RDoc::Parser
+    class Simple < Parser
 
-  include RDoc::Parser::Text
+      include Parser::Text
 
-  parse_files_matching(//)
+      parse_files_matching(//)
 
-  attr_reader :content # :nodoc:
+      attr_reader :content # :nodoc:
 
-  ##
-  # Prepare to parse a plain file
+      ##
+      # Prepare to parse a plain file
 
-  def initialize(top_level, content, options, stats)
-    super
+      def initialize(top_level, content, options, stats)
+        super
 
-    preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+        preprocess = Markup::PreProcess.new @file_name, @options.rdoc_include
 
-    content = RDoc::Text.expand_tabs(@content)
-    @content, = preprocess.run_pre_processes(content, @top_level, 1, :simple)
-  end
+        content = ::RDoc::Text.expand_tabs(@content)
+        @content, = preprocess.run_pre_processes(content, @top_level, 1, :simple)
+      end
 
-  ##
-  # Extract the file contents and attach them to the TopLevel as a comment
+      ##
+      # Extract the file contents and attach them to the TopLevel as a comment
 
-  def scan
-    content = remove_coding_comment @content
+      def scan
+        content = remove_coding_comment @content
 
-    comment = RDoc::Comment.new content, @top_level
+        comment = Comment.new content, @top_level
 
-    @top_level.comment = comment
-    @top_level
-  end
+        @top_level.comment = comment
+        @top_level
+      end
 
-  ##
-  # Removes the encoding magic comment from +text+
+      ##
+      # Removes the encoding magic comment from +text+
 
-  def remove_coding_comment(text)
-    text.sub(/\A# .*coding[=:].*$/, '')
+      def remove_coding_comment(text)
+        text.sub(/\A# .*coding[=:].*$/, '')
+      end
+    end
   end
 end

data/lib/rdoc/parser/text.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
-##
-# Indicates this parser is text and doesn't contain code constructs.
-#
-# Include this module in a RDoc::Parser subclass to make it show up as a file,
-# not as part of a class or module.
-#--
-# This is not named File to avoid overriding ::File
+module RDoc
+  class Parser
+    ##
+    # Indicates this parser is text and doesn't contain code constructs.
+    #
+    # Include this module in a RDoc::Parser subclass to make it show up as a file,
+    # not as part of a class or module.
+    #--
+    # This is not named File to avoid overriding ::File
 
-module RDoc::Parser::Text
+    module Text
+    end
+  end
 end