ruby-lsp 0.23.15 → 0.26.9

data/lib/ruby_lsp/document.rb

@@ -2,29 +2,16 @@
 # frozen_string_literal: true
 
 module RubyLsp
+  # @abstract
+  #: [ParseResultType]
   class Document
-    class LanguageId < T::Enum
-      enums do
-        Ruby = new("ruby")
-        ERB = new("erb")
-        RBS = new("rbs")
-      end
-    end
-
-    extend T::Sig
-    extend T::Helpers
-    extend T::Generic
-
-    class LocationNotFoundError < StandardError; end
-    ParseResultType = type_member
+    class InvalidLocationError < StandardError; end
 
     # This maximum number of characters for providing expensive features, like semantic highlighting and diagnostics.
     # This is the same number used by the TypeScript extension in VS Code
     MAXIMUM_CHARACTERS_FOR_EXPENSIVE_FEATURES = 100_000
     EMPTY_CACHE = Object.new.freeze #: Object
 
-    abstract!
-
     #: ParseResultType
     attr_reader :parse_result
 
@@ -56,8 +43,13 @@ module RubyLsp
       @encoding = global_state.encoding #: Encoding
       @uri = uri #: URI::Generic
       @needs_parsing = true #: bool
-      @parse_result = T.unsafe(nil) #: ParseResultType
       @last_edit = nil #: Edit?
+
+      # Workaround to be able to type parse_result properly. It is immediately set when invoking parse!
+      @parse_result = ( # rubocop:disable Style/RedundantParentheses
+        nil #: as untyped
+      ) #: ParseResultType
+
       parse!
     end
 
@@ -66,8 +58,11 @@ module RubyLsp
       self.class == other.class && uri == other.uri && @source == other.source
     end
 
-    sig { abstract.returns(LanguageId) }
-    def language_id; end
+    # @abstract
+    #: -> Symbol
+    def language_id
+      raise AbstractMethodInvokedError
+    end
 
     #: [T] (String request_name) { (Document[ParseResultType] document) -> T } -> T
     def cache_fetch(request_name, &block)
@@ -89,6 +84,11 @@ module RubyLsp
       @cache[request_name]
     end
 
+    #: (String request_name) -> void
+    def clear_cache(request_name)
+      @cache[request_name] = EMPTY_CACHE
+    end
+
     #: (Array[Hash[Symbol, untyped]] edits, version: Integer) -> void
     def push_edits(edits, version:)
       edits.each do |edit|
@@ -120,11 +120,17 @@ module RubyLsp
     end
 
     # Returns `true` if the document was parsed and `false` if nothing needed parsing
-    sig { abstract.returns(T::Boolean) }
-    def parse!; end
+    # @abstract
+    #: -> bool
+    def parse!
+      raise AbstractMethodInvokedError
+    end
 
-    sig { abstract.returns(T::Boolean) }
-    def syntax_error?; end
+    # @abstract
+    #: -> bool
+    def syntax_error?
+      raise AbstractMethodInvokedError
+    end
 
     #: -> bool
     def past_expensive_limit?
@@ -133,27 +139,28 @@ module RubyLsp
 
     #: (Hash[Symbol, untyped] start_pos, ?Hash[Symbol, untyped]? end_pos) -> [Integer, Integer?]
     def find_index_by_position(start_pos, end_pos = nil)
-      @global_state.synchronize do
-        scanner = create_scanner
-        start_index = scanner.find_char_position(start_pos)
-        end_index = scanner.find_char_position(end_pos) if end_pos
-        [start_index, end_index]
-      end
+      scanner = create_scanner
+      start_index = scanner.find_char_position(start_pos)
+      end_index = scanner.find_char_position(end_pos) if end_pos
+      [start_index, end_index]
     end
 
     private
 
     #: -> Scanner
     def create_scanner
-      Scanner.new(@source, @encoding)
+      case @encoding
+      when Encoding::UTF_8
+        Utf8Scanner.new(@source)
+      when Encoding::UTF_16LE
+        Utf16Scanner.new(@source)
+      else
+        Utf32Scanner.new(@source)
+      end
     end
 
+    # @abstract
     class Edit
-      extend T::Sig
-      extend T::Helpers
-
-      abstract!
-
       #: Hash[Symbol, untyped]
       attr_reader :range
 
@@ -167,33 +174,114 @@ module RubyLsp
     class Replace < Edit; end
     class Delete < Edit; end
 
+    # Parent class for all position scanners. Scanners are used to translate a position given by the editor into a
+    # string index that we can use to find the right place in the document source. The logic for finding the correct
+    # index depends on the encoding negotiated with the editor, so we have different subclasses for each encoding.
+    # See https://microsoft.github.io/language-server-protocol/specification/#positionEncodingKind for more information
+    # @abstract
     class Scanner
-      extend T::Sig
-
       LINE_BREAK = 0x0A #: Integer
       # After character 0xFFFF, UTF-16 considers characters to have length 2 and we have to account for that
       SURROGATE_PAIR_START = 0xFFFF #: Integer
 
-      #: (String source, Encoding encoding) -> void
-      def initialize(source, encoding)
+      #: -> void
+      def initialize
         @current_line = 0 #: Integer
         @pos = 0 #: Integer
-        @source = source.codepoints #: Array[Integer]
-        @encoding = encoding
       end
 
-      # Finds the character index inside the source string for a given line and column
+      # Finds the character index inside the source string for a given line and column. This method always returns the
+      # character index regardless of whether we are searching positions based on bytes, code units, or codepoints.
+      # @abstract
+      #: (Hash[Symbol, untyped] position) -> Integer
+      def find_char_position(position)
+        raise AbstractMethodInvokedError
+      end
+    end
+
+    # For the UTF-8 encoding, positions correspond to bytes
+    class Utf8Scanner < Scanner
+      #: (String source) -> void
+      def initialize(source)
+        super()
+        @bytes = source.bytes #: Array[Integer]
+        @character_length = 0 #: Integer
+      end
+
+      # @override
+      #: (Hash[Symbol, untyped] position) -> Integer
+      def find_char_position(position)
+        # Each group of bytes is a character. We advance based on the number of bytes to count how many full characters
+        # we have in the requested offset
+        until @current_line == position[:line]
+          byte = @bytes[@pos] #: Integer?
+          raise InvalidLocationError unless byte
+
+          until LINE_BREAK == byte
+            @pos += character_byte_length(byte)
+            @character_length += 1
+            byte = @bytes[@pos]
+            raise InvalidLocationError unless byte
+          end
+
+          @pos += 1
+          @character_length += 1
+          @current_line += 1
+        end
+
+        # @character_length has the number of characters until the beginning of the line. We don't accumulate on it for
+        # the character part because locating the same position twice must return the same value
+        line_byte_offset = 0
+        line_characters = 0
+
+        while line_byte_offset < position[:character]
+          byte = @bytes[@pos + line_byte_offset] #: Integer?
+          raise InvalidLocationError unless byte
+
+          line_byte_offset += character_byte_length(byte)
+          line_characters += 1
+        end
+
+        @character_length + line_characters
+      end
+
+      private
+
+      #: (Integer) -> Integer
+      def character_byte_length(byte)
+        if byte < 0x80 # 1-byte character
+          1
+        elsif byte < 0xE0 # 2-byte character
+          2
+        elsif byte < 0xF0 # 3-byte character
+          3
+        else # 4-byte character
+          4
+        end
+      end
+    end
+
+    # For the UTF-16 encoding, positions correspond to UTF-16 code units, which count characters beyond the surrogate
+    # pair as length 2
+    class Utf16Scanner < Scanner
+      #: (String) -> void
+      def initialize(source)
+        super()
+        @codepoints = source.codepoints #: Array[Integer]
+      end
+
+      # @override
       #: (Hash[Symbol, untyped] position) -> Integer
       def find_char_position(position)
         # Find the character index for the beginning of the requested line
         until @current_line == position[:line]
-          until LINE_BREAK == @source[@pos]
-            @pos += 1
+          codepoint = @codepoints[@pos] #: Integer?
+          raise InvalidLocationError unless codepoint
 
-            if @pos >= @source.length
-              # Pack the code points back into the original string to provide context in the error message
-              raise LocationNotFoundError, "Requested position: #{position}\nSource:\n\n#{@source.pack("U*")}"
-            end
+          until LINE_BREAK == @codepoints[@pos]
+            @pos += 1
+            codepoint = @codepoints[@pos] #: Integer?
+            raise InvalidLocationError unless codepoint
           end
 
           @pos += 1
@@ -202,29 +290,53 @@ module RubyLsp
 
         # The final position is the beginning of the line plus the requested column. If the encoding is UTF-16, we also
         # need to adjust for surrogate pairs
-        requested_position = @pos + position[:character]
+        line_characters = 0
+        line_code_units = 0
+
+        while line_code_units < position[:character]
+          code_point = @codepoints[@pos + line_characters]
+          raise InvalidLocationError unless code_point
+
+          line_code_units += if code_point > SURROGATE_PAIR_START
+            2 # Surrogate pair, so we skip the next code unit
+          else
+            1 # Single code unit character
+          end
 
-        if @encoding == Encoding::UTF_16LE
-          requested_position -= utf_16_character_position_correction(@pos, requested_position)
+          line_characters += 1
         end
 
-        requested_position
+        @pos + line_characters
+      end
+    end
+
+    # For the UTF-32 encoding, positions correspond directly to codepoints
+    class Utf32Scanner < Scanner
+      #: (String) -> void
+      def initialize(source)
+        super()
+        @codepoints = source.codepoints #: Array[Integer]
       end
 
-      # Subtract 1 for each character after 0xFFFF in the current line from the column position, so that we hit the
-      # right character in the UTF-8 representation
-      #: (Integer current_position, Integer requested_position) -> Integer
-      def utf_16_character_position_correction(current_position, requested_position)
-        utf16_unicode_correction = 0
+      # @override
+      #: (Hash[Symbol, untyped] position) -> Integer
+      def find_char_position(position)
+        # Find the character index for the beginning of the requested line
+        until @current_line == position[:line]
+          codepoint = @codepoints[@pos] #: Integer?
+          raise InvalidLocationError unless codepoint
 
-        until current_position == requested_position
-          codepoint = @source[current_position]
-          utf16_unicode_correction += 1 if codepoint && codepoint > SURROGATE_PAIR_START
+          until LINE_BREAK == @codepoints[@pos]
+            @pos += 1
+            codepoint = @codepoints[@pos] #: Integer?
+            raise InvalidLocationError unless codepoint
+          end
 
-          current_position += 1
+          @pos += 1
+          @current_line += 1
         end
 
-        utf16_unicode_correction
+        @pos + position[:character]
       end
     end
   end

data/lib/ruby_lsp/erb_document.rb

@@ -2,11 +2,8 @@
 # frozen_string_literal: true
 
 module RubyLsp
+  #: [ParseResultType = Prism::ParseLexResult]
   class ERBDocument < Document
-    extend T::Generic
-
-    ParseResultType = type_member { { fixed: Prism::ParseResult } }
-
     #: String
     attr_reader :host_language_source
 
@@ -34,11 +31,16 @@ module RubyLsp
       @host_language_source = scanner.host_language
       # Use partial script to avoid syntax errors in ERB files where keywords may be used without the full context in
       # which they will be evaluated
-      @parse_result = Prism.parse(scanner.ruby, partial_script: true)
+      @parse_result = Prism.parse_lex(scanner.ruby, partial_script: true)
       @code_units_cache = @parse_result.code_units_cache(@encoding)
       true
     end
 
+    #: -> Prism::ProgramNode
+    def ast
+      @parse_result.value.first
+    end
+
     # @override
     #: -> bool
     def syntax_error?
@@ -46,9 +48,9 @@ module RubyLsp
     end
 
     # @override
-    #: -> LanguageId
+    #: -> Symbol
     def language_id
-      LanguageId::ERB
+      :erb
     end
 
     #: (Hash[Symbol, untyped] position, ?node_types: Array[singleton(Prism::Node)]) -> NodeContext
@@ -56,7 +58,7 @@ module RubyLsp
       char_position, _ = find_index_by_position(position)
 
       RubyDocument.locate(
-        @parse_result.value,
+        ast,
         char_position,
         code_units_cache: @code_units_cache,
         node_types: node_types,

data/lib/ruby_lsp/global_state.rb

@@ -2,6 +2,21 @@
 # frozen_string_literal: true
 
 module RubyLsp
+  # Holds the detected value and the reason for detection
+  class DetectionResult
+    #: String
+    attr_reader :value
+
+    #: String
+    attr_reader :reason
+
+    #: (String value, String reason) -> void
+    def initialize(value, reason)
+      @value = value
+      @reason = reason
+    end
+  end
+
   class GlobalState
     #: String
     attr_reader :test_library
@@ -56,6 +71,17 @@ module RubyLsp
       @enabled_feature_flags = {} #: Hash[Symbol, bool]
       @mutex = Mutex.new #: Mutex
       @telemetry_machine_id = nil #: String?
+      @feature_configuration = {
+        inlayHint: RequestConfig.new({
+          enableAll: false,
+          implicitRescue: false,
+          implicitHashValue: false,
+        }),
+        codeLens: RequestConfig.new({
+          enableAll: false,
+          enableTestCodeLens: true,
+        }),
+      } #: Hash[Symbol, RequestConfig]
     end
 
     #: [T] { -> T } -> T
@@ -111,8 +137,11 @@ module RubyLsp
       end
 
       if @formatter == "auto"
-        @formatter = detect_formatter(direct_dependencies, all_dependencies)
-        notifications << Notification.window_log_message("Auto detected formatter: #{@formatter}")
+        formatter_result = detect_formatter(direct_dependencies, all_dependencies)
+        @formatter = formatter_result.value
+        notifications << Notification.window_log_message(
+          "Auto detected formatter: #{@formatter} (#{formatter_result.reason})",
+        )
       end
 
       specified_linters = options.dig(:initializationOptions, :linters)
@@ -133,21 +162,28 @@ module RubyLsp
         specified_linters << "rubocop_internal"
       end
 
-      @linters = specified_linters || detect_linters(direct_dependencies, all_dependencies)
-
-      notifications << if specified_linters
-        Notification.window_log_message("Using linters specified by user: #{@linters.join(", ")}")
+      if specified_linters
+        @linters = specified_linters
+        notifications << Notification.window_log_message("Using linters specified by user: #{@linters.join(", ")}")
       else
-        Notification.window_log_message("Auto detected linters: #{@linters.join(", ")}")
+        linter_results = detect_linters(direct_dependencies, all_dependencies)
+        @linters = linter_results.map(&:value)
+        linter_messages = linter_results.map { |r| "#{r.value} (#{r.reason})" }
+        notifications << Notification.window_log_message("Auto detected linters: #{linter_messages.join(", ")}")
       end
 
-      @test_library = detect_test_library(direct_dependencies)
-      notifications << Notification.window_log_message("Detected test library: #{@test_library}")
+      test_library_result = detect_test_library(direct_dependencies)
+      @test_library = test_library_result.value
+      notifications << Notification.window_log_message(
+        "Detected test library: #{@test_library} (#{test_library_result.reason})",
+      )
 
-      @has_type_checker = detect_typechecker(all_dependencies)
-      if @has_type_checker
+      typechecker_result = detect_typechecker(all_dependencies)
+      @has_type_checker = !typechecker_result.nil?
+      if typechecker_result
         notifications << Notification.window_log_message(
-          "Ruby LSP detected this is a Sorbet project and will defer to the Sorbet LSP for some functionality",
+          "Ruby LSP detected this is a Sorbet project (#{typechecker_result.reason}) and will defer to the " \
+            "Sorbet LSP for some functionality",
         )
       end
 
@@ -175,9 +211,19 @@ module RubyLsp
       @enabled_feature_flags = enabled_flags if enabled_flags
 
       @telemetry_machine_id = options.dig(:initializationOptions, :telemetryMachineId)
+
+      options.dig(:initializationOptions, :featuresConfiguration)&.each do |feature_name, config|
+        @feature_configuration[feature_name]&.merge!(config)
+      end
+
       notifications
     end
 
+    #: (Symbol) -> RequestConfig?
+    def feature_configuration(feature_name)
+      @feature_configuration[feature_name]
+    end
+
     #: (Symbol flag) -> bool?
     def enabled_feature?(flag)
       @enabled_feature_flags[:all] || @enabled_feature_flags[flag]
@@ -207,60 +253,67 @@ module RubyLsp
 
     private
 
-    #: (Array[String] direct_dependencies, Array[String] all_dependencies) -> String
+    #: (Array[String] direct_dependencies, Array[String] all_dependencies) -> DetectionResult
     def detect_formatter(direct_dependencies, all_dependencies)
       # NOTE: Intentionally no $ at end, since we want to match rubocop-shopify, etc.
-      return "rubocop_internal" if direct_dependencies.any?(/^rubocop/)
+      if direct_dependencies.any?(/^rubocop/)
+        return DetectionResult.new("rubocop_internal", "direct dependency matching /^rubocop/")
+      end
 
-      syntax_tree_is_direct_dependency = direct_dependencies.include?("syntax_tree")
-      return "syntax_tree" if syntax_tree_is_direct_dependency
+      if direct_dependencies.include?("syntax_tree")
+        return DetectionResult.new("syntax_tree", "direct dependency")
+      end
 
-      rubocop_is_transitive_dependency = all_dependencies.include?("rubocop")
-      return "rubocop_internal" if dot_rubocop_yml_present && rubocop_is_transitive_dependency
+      if all_dependencies.include?("rubocop") && dot_rubocop_yml_present
+        return DetectionResult.new("rubocop_internal", "transitive dependency with .rubocop.yml present")
+      end
 
-      "none"
+      DetectionResult.new("none", "no formatter detected")
     end
 
     # Try to detect if there are linters in the project's dependencies. For auto-detection, we always only consider a
     # single linter. To have multiple linters running, the user must configure them manually
-    #: (Array[String] dependencies, Array[String] all_dependencies) -> Array[String]
+    #: (Array[String] dependencies, Array[String] all_dependencies) -> Array[DetectionResult]
     def detect_linters(dependencies, all_dependencies)
-      linters = []
+      linters = [] #: Array[DetectionResult]
 
-      if dependencies.any?(/^rubocop/) || (all_dependencies.include?("rubocop") && dot_rubocop_yml_present)
-        linters << "rubocop_internal"
+      if dependencies.any?(/^rubocop/)
+        linters << DetectionResult.new("rubocop_internal", "direct dependency matching /^rubocop/")
+      elsif all_dependencies.include?("rubocop") && dot_rubocop_yml_present
+        linters << DetectionResult.new("rubocop_internal", "transitive dependency with .rubocop.yml present")
       end
 
       linters
     end
 
-    #: (Array[String] dependencies) -> String
+    #: (Array[String] dependencies) -> DetectionResult
     def detect_test_library(dependencies)
       if dependencies.any?(/^rspec/)
-        "rspec"
+        DetectionResult.new("rspec", "direct dependency matching /^rspec/")
       # A Rails app may have a dependency on minitest, but we would instead want to use the Rails test runner provided
       # by ruby-lsp-rails. A Rails app doesn't need to depend on the rails gem itself, individual components like
       # activestorage may be added to the gemfile so that other components aren't downloaded. Check for the presence
       #  of bin/rails to support these cases.
       elsif bin_rails_present
-        "rails"
+        DetectionResult.new("rails", "bin/rails present")
       # NOTE: Intentionally ends with $ to avoid mis-matching minitest-reporters, etc. in a Rails app.
       elsif dependencies.any?(/^minitest$/)
-        "minitest"
+        DetectionResult.new("minitest", "direct dependency matching /^minitest$/")
       elsif dependencies.any?(/^test-unit/)
-        "test-unit"
+        DetectionResult.new("test-unit", "direct dependency matching /^test-unit/")
       else
-        "unknown"
+        DetectionResult.new("unknown", "no test library detected")
       end
     end
 
-    #: (Array[String] dependencies) -> bool
+    #: (Array[String] dependencies) -> DetectionResult?
     def detect_typechecker(dependencies)
-      return false if ENV["RUBY_LSP_BYPASS_TYPECHECKER"]
+      return if ENV["RUBY_LSP_BYPASS_TYPECHECKER"]
+      return if dependencies.none?(/^sorbet-static/)
 
-      dependencies.any?(/^sorbet-static/)
+      DetectionResult.new("sorbet", "sorbet-static in dependencies")
     rescue Bundler::GemfileNotFound
-      false
+      nil
     end
 
     #: -> bool

data/lib/ruby_lsp/internal.rb

@@ -6,15 +6,18 @@
 yarp_require_paths = Gem.loaded_specs["yarp"]&.full_require_paths
 $LOAD_PATH.delete_if { |path| yarp_require_paths.include?(path) } if yarp_require_paths
 
-require "sorbet-runtime"
-
 # Set Bundler's UI level to silent as soon as possible to prevent any prints to STDOUT
 require "bundler"
 Bundler.ui.level = :silent
 
 require "json"
 require "uri"
-require "cgi"
+require "cgi/escape"
+if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.5")
+  # Just requiring `cgi/escape` leaves CGI.unescape broken on older rubies
+  # Some background on why this is necessary: https://bugs.ruby-lang.org/issues/21258
+  require "cgi/util"
+end
 require "set"
 require "strscan"
 require "prism"