steep 2.0.0 → 2.1.0

data/lib/steep/server/master.rb

@@ -186,6 +186,12 @@ module Steep
       attr_accessor :typecheck_automatically
       attr_reader :start_type_checking_queue
 
+      # Type check requests waiting for the current type check to finish
+      attr_reader :pending_typecheck_requests
+
+      # Callbacks to be called when no type check is running anymore
+      attr_reader :typecheck_quiescent_callbacks
+
       def initialize(project:, reader:, writer:, interaction_worker:, typecheck_workers:, queue: Queue.new, refork: false)
         @project = project
         @reader = reader
@@ -199,6 +205,11 @@ module Steep
         @write_queue = SizedQueue.new(100)
         @refork_mutex = Mutex.new
         @need_to_refork = refork
+        @typecheck_quiescent_callbacks = []
+        @pending_typecheck_requests = []
+        @project_file_mtimes = nil
+        @command_socket_requests = {}
+        @command_socket_mutex = Mutex.new
 
         @controller = TypeCheckController.new(project: project)
         @result_controller = ResultController.new()
@@ -245,7 +256,7 @@ module Steep
                 case job.dest
                 when :client
                   Steep.logger.info { "Processing SendMessageJob: dest=client, method=#{job.message[:method] || "-"}, id=#{job.message[:id] || "-"}" }
-                  writer.write job.message
+                  write_message_to_client(job.message)
                 when WorkerProcess
                   refork_mutex.synchronize do
                     Steep.logger.info { "Processing SendMessageJob: dest=#{job.dest.name}, method=#{job.message[:method] || "-"}, id=#{job.message[:id] || "-"}" }
@@ -420,6 +431,8 @@ module Steep
                 progress.end()
               end
 
+              reset_project_file_mtimes()
+
               if file_system_watcher_supported?
                 setup_file_system_watcher()
               end
@@ -445,6 +458,7 @@ module Steep
 
             unless controller.open_paths.include?(path)
               updated_watched_files << path
+              record_project_file_mtime(path)
 
               case type
               when LSP::Constant::FileChangeType::CREATED, LSP::Constant::FileChangeType::CHANGED
@@ -684,7 +698,7 @@ module Steep
             request.inline_paths << [target_name.to_sym, Pathname(path)]
           end
 
-          start_type_check(request: request, last_request: nil)
+          start_type_check(request: request, last_request: current_type_check_request)
 
         when CustomMethods::TypeCheckGroups::METHOD
           params = message[:params] #: CustomMethods::TypeCheckGroups::params
@@ -703,6 +717,15 @@ module Steep
           request.needs_response = false
           start_type_check(request: request, last_request: current_type_check_request, report_progress_threshold: 0)
 
+        when CustomMethods::Query__Diagnostics::METHOD
+          id = message[:id]
+          params = message[:params] #: CustomMethods::Query__Diagnostics::params
+          paths = params[:paths]
+
+          run_when_typecheck_quiescent do
+            collect_query_diagnostics(id, paths)
+          end
+
         when "$/ping"
           enqueue_write_job SendMessageJob.to_client(
             message: {
@@ -805,6 +828,17 @@ module Steep
 
       def start_type_check(request: nil, last_request:, progress: nil, include_unchanged: false, report_progress_threshold: 10, needs_response: nil)
         Steep.logger.tagged "#start_type_check(#{progress&.guid || request&.guid}, #{last_request&.guid}" do
+          if (current = current_type_check_request) && typecheck_request_from_command_socket?(current)
+            # Never supersede a type check that a command socket client is waiting for
+            if request
+              Steep.logger.info { "Queueing type check request #{request.guid} until the command socket request finishes" }
+              pending_typecheck_requests << request
+            else
+              Steep.logger.info { "Deferring automatic type checking until the command socket request finishes" }
+            end
+            return
+          end
+
           if last_request
             finish_type_check(last_request)
           end
@@ -884,6 +918,7 @@ module Steep
               finish_type_check(current)
               @current_type_check_request = nil
               refork_workers
+              start_pending_typecheck()
             end
           end
         end
@@ -955,6 +990,17 @@ module Steep
               refork_finished.pop
             end
           end
+
+          # The reforked workers started from a copy of the primary's state, which has the
+          # results of the primary's assigned paths only, and the results the replaced workers
+          # had computed are gone with them. Type check everything again so that queries
+          # reading the stored results, like `$/steep/query/diagnostics`, see all files.
+          job_queue << -> do
+            guid = SecureRandom.uuid
+            request = controller.make_all_request(guid: guid, progress: work_done_progress(guid))
+            request.needs_response = false
+            start_type_check(request: request, last_request: current_type_check_request, report_progress_threshold: 0)
+          end
         end
       end
 
@@ -1021,6 +1067,341 @@ module Steep
         write_queue.push(job)
       end
 
+      # Calls the block once no type check is running
+      #
+      # Starts a type check for dirty paths first, so that the block observes up-to-date results.
+      #
+      def run_when_typecheck_quiescent(&block)
+        unless current_type_check_request
+          guid = SecureRandom.uuid
+          if request = controller.make_request(guid: guid, progress: work_done_progress(guid))
+            request.needs_response = false
+            start_type_check(request: request, last_request: nil)
+          end
+        end
+
+        if current_type_check_request || !pending_typecheck_requests.empty?
+          typecheck_quiescent_callbacks << block
+        else
+          yield
+        end
+      end
+
+      # Starts the next queued type check request if any, or the automatic type check for dirty paths
+      #
+      # Calls the quiescent callbacks when nothing is left to type check.
+      #
+      def start_pending_typecheck
+        while request = pending_typecheck_requests.shift
+          start_type_check(request: request, last_request: nil)
+          return if current_type_check_request
+        end
+
+        if typecheck_automatically && initialize_params
+          guid = SecureRandom.uuid
+          if request = controller.make_request(guid: guid, progress: work_done_progress(guid))
+            request.needs_response = false
+            start_type_check(request: request, last_request: nil)
+            return if current_type_check_request
+          end
+        end
+
+        flush_typecheck_quiescent_callbacks()
+      end
+
+      # Calls the waiting quiescent callbacks, once a type check for the paths that went dirty
+      # in the meantime finishes
+      #
+      def flush_typecheck_quiescent_callbacks
+        return if typecheck_quiescent_callbacks.empty?
+        return if current_type_check_request
+
+        guid = SecureRandom.uuid
+        if request = controller.make_request(guid: guid, progress: work_done_progress(guid))
+          request.needs_response = false
+          start_type_check(request: request, last_request: nil)
+          return if current_type_check_request
+        end
+
+        callbacks = typecheck_quiescent_callbacks.dup
+        typecheck_quiescent_callbacks.clear
+        callbacks.each(&:call)
+      end
+
+      # Collects the stored diagnostics from all typecheck workers and sends the response
+      #
+      # `paths` is an array of absolute path strings to filter the result, or `nil` to return everything.
+      #
+      def collect_query_diagnostics(id, paths)
+        uris = paths&.map {|path| PathHelper.to_uri(Pathname(path)).to_s }
+
+        result_controller << group_request do |group|
+          typecheck_workers.each do |worker|
+            group << send_request(method: CustomMethods::Query__Diagnostics::METHOD, params: nil, worker: worker)
+          end
+
+          group.on_completion do |handlers|
+            diagnostics = {} #: Hash[String, Array[untyped]]
+
+            handlers.each do |handler|
+              result = handler.result or next
+              result.each do |entry|
+                array = diagnostics[entry[:uri]] ||= []
+                array.concat(entry[:diagnostics] || [])
+                array.uniq!
+              end
+            end
+
+            # @type var result: CustomMethods::Query__Diagnostics::result
+            result =
+              if uris
+                # Files the server has not type checked yet are reported with `diagnostics: nil`
+                uris.sort.map do |uri|
+                  { uri: uri, diagnostics: diagnostics[uri] }
+                end
+              else
+                diagnostics.keys.sort.map do |uri|
+                  { uri: uri, diagnostics: diagnostics.fetch(uri) }
+                end
+              end
+
+            enqueue_write_job SendMessageJob.to_client(
+              message: CustomMethods::Query__Diagnostics.response(id, result)
+            )
+          end
+        end
+      end
+
+      # Methods that command socket clients cannot send because they control the server lifecycle
+      # The only methods a command socket client may send: what `steep check` and
+      # `steep query` use. Everything else is rejected -- lifecycle methods because the
+      # server belongs to whoever started it, and document synchronization because the
+      # LSP client owns the content of the files it opens.
+      COMMAND_SOCKET_ALLOWED_METHODS = [
+        CustomMethods::TypeCheck::METHOD,
+        CustomMethods::Query__Diagnostics::METHOD,
+        CustomMethods::Query__Definition::METHOD,
+        "textDocument/hover",
+        "$/ping"
+      ].freeze
+
+      # Notifications that are copied to command socket sessions with a pending `$/steep/typecheck` request
+      COMMAND_SOCKET_FORWARDED_NOTIFICATIONS = ["textDocument/publishDiagnostics", "window/showMessage"].freeze
+
+      # Processes a message received from a command socket client
+      #
+      # Requests are assigned a fresh id and enqueued as if they came from the LSP client,
+      # and the responses are routed back to the session through `#write_message_to_client`.
+      #
+      # May be called from any thread.
+      #
+      def process_command_socket_message(message, session)
+        method = message[:method]
+        id = message[:id]
+
+        if method && !COMMAND_SOCKET_ALLOWED_METHODS.include?(method.to_s)
+          Steep.logger.warn { "Command socket: rejected `#{method}`" }
+          if id
+            session.write({
+              id: id,
+              error: {
+                code: LSP::Constant::ErrorCodes::INVALID_REQUEST,
+                message: "`#{method}` is not allowed through the command socket"
+              }
+            })
+          end
+          return
+        end
+
+        unless initialize_params
+          if id
+            session.write({
+              id: id,
+              error: {
+                code: LSP::Constant::ErrorCodes::SERVER_NOT_INITIALIZED,
+                message: "The language server is not initialized yet"
+              }
+            })
+          end
+          return
+        end
+
+        case
+        when method && id
+          guid = SecureRandom.uuid
+          @command_socket_mutex.synchronize do
+            @command_socket_requests[guid] = [session, id, method.to_s == CustomMethods::TypeCheck::METHOD]
+          end
+          message = message.merge({ id: guid })
+        when id
+          # A response from the client, but nothing is waiting for it
+          return
+        end
+
+        # Reload files changed on disk before processing, because command socket clients
+        # modify files without sending `didChange` notifications
+        job_queue << -> do
+          sync_project_files_from_disk()
+          process_message_from_client(message)
+        end
+      rescue ClosedQueueError
+        Steep.logger.warn { "Command socket: server is shutting down" }
+      end
+
+      # Deregisters the pending requests of a disconnected command socket session
+      def finish_command_socket_session(session)
+        @command_socket_mutex.synchronize do
+          @command_socket_requests.delete_if {|_, entry| entry[0] == session }
+        end
+      end
+
+      def write_message_to_client(message)
+        unless message.key?(:method)
+          entry = @command_socket_mutex.synchronize { @command_socket_requests.delete(message[:id]) }
+          if entry
+            session, original_id, _ = entry
+            session.write(message.merge({ id: original_id }))
+            return
+          end
+        end
+
+        writer.write(message)
+
+        if message.key?(:method) && !message.key?(:id)
+          forward_notification_to_command_sessions(message)
+        end
+      end
+
+      def forward_notification_to_command_sessions(message)
+        return unless COMMAND_SOCKET_FORWARDED_NOTIFICATIONS.include?(message[:method].to_s)
+
+        sessions = @command_socket_mutex.synchronize do
+          @command_socket_requests.each_value.with_object([]) do |entry, array|
+            session, _, typecheck = entry
+            array << session if typecheck
+          end
+        end
+
+        sessions.uniq!
+        sessions.each {|session| session.write(message) }
+      end
+
+      def typecheck_request_from_command_socket?(request)
+        @command_socket_mutex.synchronize do
+          @command_socket_requests.key?(request.guid)
+        end
+      end
+
+      # Records the mtimes of all project files, as the baseline of `#sync_project_files_from_disk`
+      def reset_project_file_mtimes
+        mtimes = {} #: Hash[Pathname, Time?]
+        each_project_file_path do |path|
+          mtimes[path] = file_mtime(path)
+        end
+        @project_file_mtimes = mtimes
+      end
+
+      def record_project_file_mtime(path)
+        if mtimes = @project_file_mtimes
+          mtimes[path] = file_mtime(path)
+        end
+      end
+
+      def file_mtime(path)
+        File.mtime(path)
+      rescue SystemCallError
+        nil
+      end
+
+      def each_project_file_path(&block)
+        loader = Services::FileLoader.new(base_dir: project.base_dir)
+        project.targets.each do |target|
+          loader.each_path_in_target(target) do |relative_path|
+            yield project.absolute_path(relative_path)
+          end
+        end
+      end
+
+      # Detects files changed on disk since the last load/sync, and reloads them
+      #
+      # Skips files open in the editor because the client owns their contents.
+      # Does nothing until the initial project load finishes.
+      #
+      def sync_project_files_from_disk
+        mtimes = @project_file_mtimes or return
+
+        changes = {} #: Hash[Pathname, Symbol]
+        seen = Set[] #: Set[Pathname]
+
+        each_project_file_path do |path|
+          next if seen.include?(path)
+          seen << path
+
+          mtime = file_mtime(path)
+          recorded = mtimes[path]
+
+          case
+          when recorded.nil? && mtime
+            changes[path] = mtimes.key?(path) ? :changed : :created
+          when recorded && mtime.nil?
+            changes[path] = :deleted
+          when recorded != mtime
+            changes[path] = :changed
+          end
+
+          mtimes[path] = mtime
+        end
+
+        deleted_paths = mtimes.keys.select {|path| !seen.include?(path) && mtimes[path] && file_mtime(path).nil? }
+        deleted_paths.each do |path|
+          changes[path] = :deleted
+          mtimes[path] = nil
+        end
+
+        changes.delete_if {|path, _| controller.open_paths.include?(path) }
+        return if changes.empty?
+
+        Steep.logger.info { "Command socket: reloading #{changes.size} file(s) changed on disk" }
+
+        changes.each do |path, type|
+          content =
+            if type == :deleted
+              ""
+            else
+              begin
+                path.read
+              rescue SystemCallError
+                next
+              end
+            end
+
+          case
+          when controller.code_path?(path)
+            controller.add_dirty_code_path(path)
+          when controller.signature_path?(path)
+            controller.add_dirty_signature_path(path)
+          when controller.inline_path?(path)
+            controller.add_dirty_inline_path(path, content)
+          end
+
+          broadcast_notification(CustomMethods::FileReset.notification({ uri: PathHelper.to_uri(path).to_s, content: content }))
+        end
+
+        if typecheck_automatically
+          start_type_checking_queue.execute do
+            job_queue.push(
+              -> do
+                start_type_check(
+                  last_request: current_type_check_request,
+                  progress: work_done_progress(SecureRandom.uuid),
+                  needs_response: false
+                )
+              end
+            )
+          end
+        end
+      end
+
       def work_done_progress(guid)
         if work_done_progress_supported?
           WorkDoneProgress.new(guid) do |message|

data/lib/steep/server/type_check_worker.rb

@@ -1,13 +1,14 @@
 module Steep
   module Server
     class TypeCheckWorker < BaseWorker
-      attr_reader :project, :assignment, :service
+      attr_reader :project, :assignment
       attr_reader :commandline_args
       attr_reader :current_type_check_guid
 
       WorkspaceSymbolJob = _ = Struct.new(:query, :id, keyword_init: true)
       StatsJob = _ = Struct.new(:id, keyword_init: true)
       QueryDefinitionJob = _ = Struct.new(:id, :name, keyword_init: true)
+      QueryDiagnosticsJob = _ = Struct.new(:id, keyword_init: true)
       StartTypeCheckJob = _ = Struct.new(:guid, :changes, keyword_init: true)
       TypeCheckCodeJob = _ = Struct.new(:guid, :path, :target, keyword_init: true)
       ValidateAppSignatureJob = _ = Struct.new(:guid, :path, :target, keyword_init: true)
@@ -68,7 +69,7 @@ module Steep
         @io_socket = io_socket
         @service = service if service
         @child_pids = []
-        @need_to_warmup = defined?(Process.warmup)
+        @need_to_warmup = true
 
         if io_socket
           Signal.trap "SIGCHLD" do
@@ -112,6 +113,8 @@ module Steep
         when CustomMethods::Query__Definition::METHOD
           params = request[:params] #: CustomMethods::Query__Definition::params
           queue << QueryDefinitionJob.new(id: request[:id], name: params[:name])
+        when CustomMethods::Query__Diagnostics::METHOD
+          queue << QueryDiagnosticsJob.new(id: request[:id])
         when "textDocument/definition"
           queue << GotoJob.definition(id: request[:id], params: request[:params])
         when "textDocument/implementation"
@@ -307,6 +310,10 @@ module Steep
           writer.write(
             CustomMethods::Query__Definition.response(job.id, query_definition_result(job.name))
           )
+        when QueryDiagnosticsJob
+          writer.write(
+            CustomMethods::Query__Diagnostics.response(job.id, query_diagnostics_result())
+          )
         end
       end
 
@@ -358,6 +365,45 @@ module Steep
         end
       end
 
+      # Returns the diagnostics of the files this worker has type checked so far
+      #
+      # An array of `Query__Diagnostics::entry`, with the LSP-formatted diagnostics per file URI.
+      # Files that are loaded but not type checked yet are not included.
+      #
+      def query_diagnostics_result
+        result = {} #: Hash[String, Array[untyped]]
+
+        service.source_files.each_value do |file|
+          next if file.typing.nil? && file.errors.nil?
+
+          absolute_path = project.absolute_path(file.path)
+
+          group_target =
+            project.group_for_source_path(absolute_path) ||
+            project.group_for_inline_source_path(absolute_path) ||
+            project.target_for_source_path(absolute_path) ||
+            project.target_for_inline_source_path(absolute_path)
+          next unless group_target
+
+          formatter = Diagnostic::LSPFormatter.new(group_target.code_diagnostics_config)
+          uri = PathHelper.to_uri(absolute_path).to_s
+          array = result[uri] ||= []
+          array.concat(file.diagnostics.filter_map { formatter.format(_1) })
+        end
+
+        formatter = Diagnostic::LSPFormatter.new({}, **{})
+        service.signature_validation_diagnostics.each_value do |path_diagnostics|
+          path_diagnostics.each do |path, diagnostics|
+            absolute_path = path.absolute? ? path : project.absolute_path(path)
+            uri = PathHelper.to_uri(absolute_path).to_s
+            array = result[uri] ||= []
+            array.concat(diagnostics.filter_map { formatter.format(_1) })
+          end
+        end
+
+        result.map { { uri: _1, diagnostics: _2 } }
+      end
+
       def query_definition_result(name_string)
         name = Services::GotoService.parse_name(name_string)
 

data/lib/steep/services/goto_service.rb

@@ -422,7 +422,7 @@ module Steep
           when :send
             location = (_ = node.location) #: Parser::AST::_SelectorLocation
             if test_ast_location(location.selector, line: line, column: column)
-              if (parent = parents[0]) && parent.type == :block && parent.children[0] === node
+              if (parent = parents[0]) && (parent.type == :block || parent.type == :numblock || parent.type == :itblock) && parent.children[0] === node
                 node = parents[0]
               end
 
@@ -524,7 +524,9 @@ module Steep
       def constant_definition_in_ruby(name, locations:)
         type_check.source_files.each do |path, source|
           if typing = source.typing
-            target = project.target_for_source_path(path) or raise
+            # Inline sources are type checked too, but they don't have a Ruby definition to go to.
+            # Their declarations are found through the RBS lookup instead.
+            target = project.target_for_source_path(path) or next
             entry = typing.source_index.entry(constant: name)
             entry.definitions.each do |node|
               case node.type
@@ -551,7 +553,9 @@ module Steep
         if in_ruby
           type_check.source_files.each do |path, source|
             if typing = source.typing
-              target = project.target_for_source_path(path) or raise
+              # Inline sources are type checked too, but they don't have a Ruby definition to go to.
+              # Their declarations are found through the RBS lookup instead.
+              target = project.target_for_source_path(path) or next
               entry = typing.source_index.entry(method: name)
 
               if entry.definitions.empty?

data/lib/steep/services/hover_provider/content.rb

@@ -56,7 +56,7 @@ module Steep
 
           def class_or_module?
             (class_decl || class_alias) ?  true : false
-        end
+          end
       end
 
       TypeAliasContent = _ = Struct.new(:location, :decl, keyword_init: true)

data/lib/steep/services/hover_provider/ruby.rb

@@ -103,7 +103,7 @@ module Steep
             when :send, :csend
               result_node =
                 case parents[0]&.type
-                when :block, :numblock
+                when :block, :numblock, :itblock
                   if node == parents.fetch(0).children[0]
                     parents.fetch(0)
                   else

data/lib/steep/services/signature_help_provider.rb

@@ -59,7 +59,7 @@ module Steep
               if begin_loc.end_pos <= pos && pos <= end_loc.begin_pos
                 # Given position is between open/close parens of args of send node
 
-                if parent && (parent.type == :block || parent.type == :numblock) && node.equal?(parent.children[0])
+                if parent && (parent.type == :block || parent.type == :numblock || parent.type == :itblock) && node.equal?(parent.children[0])
                   send_node = parent
                 else
                   send_node = node

data/lib/steep/source.rb

@@ -31,7 +31,7 @@ module Steep
     end
 
     def self.new_parser
-      Prism::Translation::Parser33.new(Builder.new).tap do |parser|
+      Prism::Translation::Parser34.new(Builder.new).tap do |parser|
         parser.diagnostics.all_errors_are_fatal = true
         parser.diagnostics.ignore_warnings = true
       end
@@ -231,7 +231,7 @@ module Steep
         end
 
       when :rescue
-        body, resbodies, else_node, loc = deconstruct_rescue_node!(node)
+        body, _, else_node, loc = deconstruct_rescue_node!(node)
 
         if else_node
           loc.else or raise
@@ -257,7 +257,7 @@ module Steep
         annot.line or next
 
         case node.type
-        when :def, :module, :class, :block, :numblock, :ensure, :defs, :resbody
+        when :def, :module, :class, :block, :numblock, :itblock, :ensure, :defs, :resbody
           location = node.loc
           location.line <= annot.line && annot.line < location.last_line
         else
@@ -356,7 +356,6 @@ module Steep
 
     def find_heredoc_nodes(line, column, position)
       each_heredoc_node() do |nodes, location|
-        node = nodes[0]
         loc = location.heredoc_body #: Parser::Source::Range
 
         if range = loc.to_range
@@ -470,7 +469,7 @@ module Steep
       return false unless send_node
 
       if send_node.type == :send
-        receiver, method, args = deconstruct_send_node!(send_node)
+        receiver, method, _ = deconstruct_send_node!(send_node)
 
         return false unless receiver
 
@@ -565,13 +564,13 @@ module Steep
                   end
                 ]
               )
-            when :numblock
-              send, size, body = node.children
+            when :numblock, :itblock
+              send, arg, body = node.children
               node = node.updated(
                 nil,
                 [
                   map_child_node(send) {|child| insert_type_node(child, child_assertions) },
-                  size,
+                  arg,
                   insert_type_node(body, child_assertions)
                 ]
               )
@@ -648,7 +647,7 @@ module Steep
         case node.type
         when :send, :csend
           node
-        when :block, :numblock
+        when :block, :numblock, :itblock
           send = node.children[0]
           case send.type
           when :send, :csend
@@ -657,7 +656,7 @@ module Steep
         end
 
       if send_node
-        receiver_node, name, _, location = deconstruct_send_node!(send_node)
+        receiver_node, _, _, location = deconstruct_send_node!(send_node)
 
         if receiver_node
           if location.dot && location.selector