steep 1.9.3 → 1.10.0.dev.1

data/lib/steep/server/master.rb

@@ -178,6 +178,7 @@ module Steep
       attr_reader :job_queue, :write_queue
 
       attr_reader :current_type_check_request
+      attr_reader :refork_mutex
       attr_reader :controller
       attr_reader :result_controller
 
@@ -185,7 +186,7 @@ module Steep
       attr_accessor :typecheck_automatically
       attr_reader :start_type_checking_queue
 
-      def initialize(project:, reader:, writer:, interaction_worker:, typecheck_workers:, queue: Queue.new)
+      def initialize(project:, reader:, writer:, interaction_worker:, typecheck_workers:, queue: Queue.new, refork: false)
         @project = project
         @reader = reader
         @writer = writer
@@ -196,6 +197,8 @@ module Steep
         @commandline_args = []
         @job_queue = queue
         @write_queue = SizedQueue.new(100)
+        @refork_mutex = Mutex.new
+        @need_to_refork = refork
 
         @controller = TypeCheckController.new(project: project)
         @result_controller = ResultController.new()
@@ -244,8 +247,10 @@ module Steep
                   Steep.logger.info { "Processing SendMessageJob: dest=client, method=#{job.message[:method] || "-"}, id=#{job.message[:id] || "-"}" }
                   writer.write job.message
                 when WorkerProcess
-                  Steep.logger.info { "Processing SendMessageJob: dest=#{job.dest.name}, method=#{job.message[:method] || "-"}, id=#{job.message[:id] || "-"}" }
-                  job.dest << job.message
+                  refork_mutex.synchronize do
+                    Steep.logger.info { "Processing SendMessageJob: dest=#{job.dest.name}, method=#{job.message[:method] || "-"}, id=#{job.message[:id] || "-"}" }
+                    job.dest << job.message
+                  end
                 end
               end
             end
@@ -287,11 +292,13 @@ module Steep
             end
           end
 
-          waiter = ThreadWaiter.new
-          each_worker do |worker|
-            waiter << worker.wait_thread
+          waiter = ThreadWaiter.new(each_worker.to_a) {|worker| worker.wait_thread }
+          # @type var th: Thread & WorkerProcess::_ProcessWaitThread
+          while th = _ = waiter.wait_one()
+            if each_worker.any? { |worker| worker.pid == th.pid }
+              break # The worker unexpectedly exited
+            end
           end
-          waiter.wait_one()
 
           unless job_queue.closed?
             # Exit by error
@@ -413,49 +420,7 @@ module Steep
               end
 
               if file_system_watcher_supported?
-                patterns = [] #: Array[String]
-                project.targets.each do |target|
-                  target.source_pattern.patterns.each do |pat|
-                    path = project.base_dir + pat
-                    patterns << path.to_s unless path.directory?
-                  end
-                  target.source_pattern.prefixes.each do |pat|
-                    path = project.base_dir + pat
-                    patterns << (path + "**/*.rb").to_s unless path.file?
-                  end
-                  target.signature_pattern.patterns.each do |pat|
-                    path = project.base_dir + pat
-                    patterns << path.to_s unless path.directory?
-                  end
-                  target.signature_pattern.prefixes.each do |pat|
-                    path = project.base_dir + pat
-                    patterns << (path + "**/*.rbs").to_s unless path.file?
-                  end
-                end
-                patterns.sort!
-                patterns.uniq!
-
-                Steep.logger.info { "Setting up didChangeWatchedFiles with pattern: #{patterns.inspect}" }
-
-                enqueue_write_job SendMessageJob.to_client(
-                  message: {
-                    id: SecureRandom.uuid,
-                    method: "client/registerCapability",
-                    params: {
-                      registrations: [
-                        {
-                          id: SecureRandom.uuid,
-                          method: "workspace/didChangeWatchedFiles",
-                          registerOptions: {
-                            watchers: patterns.map do |pattern|
-                              { globPattern: pattern }
-                            end
-                          }
-                        }
-                      ]
-                    }
-                  }
-                )
+                setup_file_system_watcher()
               end
 
               controller.changed_paths.clear()
@@ -635,7 +600,7 @@ module Steep
             enqueue_write_job SendMessageJob.to_client(
               message: {
                 id: message[:id],
-                result: []
+                result: [] #: Array[untyped]
               }
             )
           end
@@ -800,7 +765,7 @@ module Steep
           Steep.logger.info "Starting new progress..."
 
           @current_type_check_request = request
-          
+
           if progress
             # If `request:` keyword arg is not given
             request.work_done_progress.begin("Type checking", request_id: fresh_request_id)
@@ -836,6 +801,76 @@ module Steep
             if current.finished?
               finish_type_check(current)
               @current_type_check_request = nil
+              refork_workers
+            end
+          end
+        end
+      end
+
+      def refork_workers
+        return unless @need_to_refork
+        @need_to_refork = false
+
+        Thread.new do
+          Thread.current.abort_on_exception = true
+
+          primary, *others = typecheck_workers
+          primary or raise
+          others.each do |worker|
+            worker.index or raise
+
+            refork_mutex.synchronize do
+              refork_finished = Thread::Queue.new
+              stdin_in, stdin_out = IO.pipe
+              stdout_in, stdout_out = IO.pipe
+
+              result_controller << send_refork_request(params: { index: worker.index, max_index: typecheck_workers.size }, worker: primary) do |handler|
+                handler.on_completion do |response|
+                  writer = LanguageServer::Protocol::Transport::Io::Writer.new(stdin_out)
+                  reader = LanguageServer::Protocol::Transport::Io::Reader.new(stdout_in)
+
+                  pid = response[:result][:pid]
+                  # It does not need to wait worker process
+                  # because the primary worker monitors it instead.
+                  #
+                  # @type var wait_thread: Thread & WorkerProcess::_ProcessWaitThread
+                  wait_thread = _ = Thread.new { sleep }
+                  wait_thread.define_singleton_method(:pid) { pid }
+
+                  new_worker = WorkerProcess.new(reader:, writer:, stderr: nil, wait_thread:, name: "#{worker.name}-2", index: worker.index)
+                  old_worker = typecheck_workers[worker.index] or raise
+
+                  typecheck_workers[(new_worker.index or raise)] = new_worker
+
+                  original_old_worker = old_worker.dup
+                  old_worker.redirect_to new_worker
+
+                  refork_finished << true
+
+                  result_controller << send_request(method: 'shutdown', worker: original_old_worker) do |handler|
+                    handler.on_completion do
+                      send_request(method: 'exit', worker: original_old_worker)
+                    end
+                  end
+
+                  Thread.new do
+                    tags = Steep.logger.formatter.current_tags.dup
+                    Steep.logger.formatter.push_tags(*tags, "from-worker@#{new_worker.name}")
+                    new_worker.reader.read do |message|
+                      job_queue << ReceiveMessageJob.new(source: new_worker, message: message)
+                    end
+                  end
+                end
+              end
+
+              # The primary worker starts forking when it receives the IOs.
+              primary.io_socket or raise
+              primary.io_socket.send_io(stdin_in)
+              primary.io_socket.send_io(stdout_out)
+              stdin_in.close
+              stdout_out.close
+
+              refork_finished.pop
             end
           end
         end
@@ -868,6 +903,25 @@ module Steep
         end
       end
 
+      def send_refork_request(id: fresh_request_id(), params:, worker:, &block)
+        method = CustomMethods::Refork::METHOD
+        Steep.logger.info "Sending request #{method}(#{id}) to #{worker.name}"
+
+        # @type var message: lsp_request
+        message = { method: method, id: id, params: params }
+        ResultHandler.new(request: message).tap do |handler|
+          yield handler if block
+
+          job = SendMessageJob.to_worker(worker, message: message)
+          case job.dest
+          when WorkerProcess
+            job.dest << job.message
+          else
+            raise "Unexpected destination: #{job.dest}"
+          end
+        end
+      end
+
       def group_request()
         GroupHandler.new().tap do |group|
           yield group
@@ -907,6 +961,58 @@ module Steep
           )
         end
       end
+
+      def setup_file_system_watcher()
+        patterns = [] #: Array[String]
+
+        project.targets.each do |target|
+          patterns.concat(paths_to_watch(target.source_pattern, extname: ".rb"))
+          patterns.concat(paths_to_watch(target.signature_pattern, extname: ".rbs"))
+          target.groups.each do |group|
+            patterns.concat(paths_to_watch(group.source_pattern, extname: ".rb"))
+            patterns.concat(paths_to_watch(group.signature_pattern, extname: ".rbs"))
+          end
+        end
+        patterns.sort!
+        patterns.uniq!
+
+        Steep.logger.info { "Setting up didChangeWatchedFiles with pattern: #{patterns.inspect}" }
+
+        enqueue_write_job SendMessageJob.to_client(
+          message: {
+            id: SecureRandom.uuid,
+            method: "client/registerCapability",
+            params: {
+              registrations: [
+                {
+                  id: SecureRandom.uuid,
+                  method: "workspace/didChangeWatchedFiles",
+                  registerOptions: {
+                    watchers: patterns.map do |pattern|
+                      { globPattern: pattern }
+                    end
+                  }
+                }
+              ]
+            }
+          }
+        )
+      end
+
+      def paths_to_watch(pattern, extname:)
+        result = [] #: Array[String]
+
+        pattern.patterns.each do |pat|
+          path = project.base_dir + pat
+          result << path.to_s unless path.directory?
+        end
+        pattern.prefixes.each do |pat|
+          path = project.base_dir + pat
+          result << (path + "**/*#{extname}").to_s unless path.file?
+        end
+
+        result
+      end
     end
   end
 end

data/lib/steep/server/type_check_worker.rb

@@ -51,15 +51,28 @@ module Steep
 
       include ChangeBuffer
 
-      def initialize(project:, reader:, writer:, assignment:, commandline_args:)
+      attr_reader :io_socket
+
+      def initialize(project:, reader:, writer:, assignment:, commandline_args:, io_socket: nil, buffered_changes: nil, service: nil)
         super(project: project, reader: reader, writer: writer)
 
         @assignment = assignment
-        @buffered_changes = {}
+        @buffered_changes = buffered_changes || {}
         @mutex = Mutex.new()
         @queue = Queue.new
         @commandline_args = commandline_args
         @current_type_check_guid = nil
+        @io_socket = io_socket
+        @service = service if service
+        @child_pids = []
+
+        if io_socket
+          Signal.trap "SIGCHLD" do
+            while pid = Process.wait(-1, Process::WNOHANG)
+              raise "Unexpected worker process exit: #{pid}" if @child_pids.include?(pid)
+            end
+          end
+        end
       end
 
       def service
@@ -98,6 +111,39 @@ module Steep
           queue << GotoJob.implementation(id: request[:id], params: request[:params])
         when "textDocument/typeDefinition"
           queue << GotoJob.type_definition(id: request[:id], params: request[:params])
+        when CustomMethods::Refork::METHOD
+          io_socket or raise
+
+          # Receive IOs before fork to avoid receiving them from multiple processes
+          stdin = io_socket.recv_io
+          stdout = io_socket.recv_io
+
+          if pid = fork
+            stdin.close
+            stdout.close
+            @child_pids << pid
+            writer.write(CustomMethods::Refork.response(request[:id], { pid: }))
+          else
+            io_socket.close
+
+            reader.close
+            writer.close
+
+            reader = LanguageServer::Protocol::Transport::Io::Reader.new(stdin)
+            writer = LanguageServer::Protocol::Transport::Io::Writer.new(stdout)
+            Steep.logger.info("Reforked worker: #{Process.pid}, params: #{request[:params]}")
+            index = request[:params][:index]
+            assignment = Services::PathAssignment.new(max_index: request[:params][:max_index], index: index)
+
+            worker = self.class.new(project: project, reader: reader, writer: writer, assignment: assignment, commandline_args: commandline_args, io_socket: nil, buffered_changes: buffered_changes, service: service)
+
+            tags = Steep.logger.formatter.current_tags.dup
+            tags[tags.find_index("typecheck:typecheck@0")] = "typecheck:typecheck@#{index}-reforked"
+            Steep.logger.formatter.push_tags(tags)
+            worker.run()
+
+            raise "unreachable"
+          end
         end
       end
 

data/lib/steep/server/worker_process.rb

@@ -8,32 +8,30 @@ module Steep
       attr_reader :name
       attr_reader :wait_thread
       attr_reader :index
+      attr_reader :io_socket
 
-      def initialize(reader:, writer:, stderr:, wait_thread:, name:, index: nil)
+      def initialize(reader:, writer:, io_socket: nil, stderr:, wait_thread:, name:, index: nil)
         @reader = reader
         @writer = writer
         @stderr = stderr
+        @io_socket = io_socket
         @wait_thread = wait_thread
         @name = name
         @index = index
       end
 
       def self.start_worker(type, name:, steepfile:, steep_command:, index: nil, delay_shutdown: false, patterns: [])
-        begin
-          unless steep_command
-            fork_worker(
-              type,
-              name: name,
-              steepfile: steepfile,
-              index: index,
-              delay_shutdown: delay_shutdown,
-              patterns: patterns
-            )
-          else
-            # Use `#spawn_worker`
-            raise NotImplementedError
-          end
-        rescue NotImplementedError
+        if Steep.can_fork? && !steep_command
+          fork_worker(
+            type,
+            name: name,
+            steepfile: steepfile,
+            index: index,
+            is_primary: index && (index[1] == 0 && index[0] >= 2),
+            delay_shutdown: delay_shutdown,
+            patterns: patterns
+          )
+        else
           spawn_worker(
             type,
             name: name,
@@ -46,9 +44,10 @@ module Steep
         end
       end
 
-      def self.fork_worker(type, name:, steepfile:, index:, delay_shutdown:, patterns:)
+      def self.fork_worker(type, name:, steepfile:, index:, delay_shutdown:, patterns:, is_primary:)
         stdin_in, stdin_out = IO.pipe
         stdout_in, stdout_out = IO.pipe
+        sock_master, sock_worker = UNIXSocket.socketpair if is_primary
 
         worker = Drivers::Worker.new(stdout: stdout_out, stdin: stdin_in, stderr: STDERR)
 
@@ -61,12 +60,14 @@ module Steep
           worker.index = this
         end
         worker.commandline_args = patterns
+        worker.io_socket = sock_worker
 
         pid = fork do
           Process.setpgid(0, 0)
           Steep.ui_logger.level = :fatal
           stdin_out.close
           stdout_in.close
+          sock_master&.close
           worker.run()
         end
 
@@ -81,6 +82,7 @@ module Steep
 
         stdin_in.close
         stdout_out.close
+        sock_worker&.close
 
         new(
           reader: reader,
@@ -88,7 +90,8 @@ module Steep
           stderr: STDERR,
           wait_thread: wait_thread,
           name: name,
-          index: index&.[](1)
+          index: index&.[](1),
+          io_socket: sock_master,
         )
       end
 
@@ -146,6 +149,10 @@ module Steep
         end
       end
 
+      def redirect_to(worker)
+        @writer = worker.writer
+      end
+
       def <<(message)
         writer.write(message)
       end
@@ -155,11 +162,11 @@ module Steep
       end
 
       def kill(force: false)
-        Steep.logger.tagged("WorkerProcess#kill@#{name}(#{wait_thread.pid})") do
+        Steep.logger.tagged("WorkerProcess#kill@#{name}(#{pid})") do
           begin
             signal = force ? :KILL : :TERM
             Steep.logger.debug("Sending signal SIG#{signal}...")
-            Process.kill(signal, wait_thread.pid)
+            Process.kill(signal, pid)
             Steep.logger.debug("Successfully sent the signal.")
           rescue Errno::ESRCH => error
             Steep.logger.debug("Failed #{error.inspect}")
@@ -171,6 +178,10 @@ module Steep
           end
         end
       end
+
+      def pid
+        wait_thread.pid
+      end
     end
   end
 end

data/lib/steep/services/completion_provider.rb

@@ -51,9 +51,29 @@ module Steep
             raise
           end
         end
+
+        def deprecated?
+          annotations =
+            case entry = env.constant_entry(full_name)
+            when RBS::Environment::ConstantEntry
+              entry.decl.annotations
+            when RBS::Environment::ClassEntry, RBS::Environment::ModuleEntry
+              entry.decls.flat_map { _1.decl.annotations }
+            when RBS::Environment::ClassAliasEntry, RBS::Environment::ModuleAliasEntry
+              entry.decl.annotations
+            else
+              raise
+            end
+            
+          if AnnotationsHelper.deprecated_annotation?(annotations)
+            true
+          else
+            false
+          end
+        end
       end
 
-      SimpleMethodNameItem = _ = Struct.new(:identifier, :range, :receiver_type, :method_types, :method_member, :method_name, keyword_init: true) do
+      SimpleMethodNameItem = _ = Struct.new(:identifier, :range, :receiver_type, :method_types, :method_member, :method_name, :deprecated, keyword_init: true) do
         # @implements SimpleMethodNameItem
 
         def comment
@@ -640,13 +660,15 @@ module Steep
                   all_members.each do |member|
                     associated_decl = all_decls.find {|decl| decl.method_def.member == member } or next
                     overloads = method_entry.overloads.select {|overload| overload.method_defs.any? {|defn| defn.member == member }}
+                    annotations = associated_decl.method_def.member_annotations
                     items << SimpleMethodNameItem.new(
                       identifier: name,
                       range: range,
                       receiver_type: type,
                       method_name: associated_decl.method_name,
                       method_types: overloads.map {|overload| subtyping.factory.method_type_1(overload.method_type) },
-                      method_member: member
+                      method_member: member,
+                      deprecated: AnnotationsHelper.deprecated_annotation?(annotations) ? true : false
                     )
                   end
                 else

data/lib/steep/services/signature_service.rb

@@ -295,7 +295,7 @@ module Steep
 
           unless errors.empty?
             # Builder won't be used.
-            factory = AST::Types::Factory.new(builder: _ = nil)
+            factory = AST::Types::Factory.new(builder: RBS::DefinitionBuilder.new(env: env, ancestor_builder: builder))
             return errors.map {|error| Diagnostic::Signature.from_rbs_error(error, factory: factory) }
           end
 

data/lib/steep/services/type_check_service.rb

@@ -302,7 +302,22 @@ module Steep
       def self.type_check(source:, subtyping:, constant_resolver:, cursor:)
         annotations = source.annotations(block: source.node, factory: subtyping.factory, context: nil)
 
-        definition = subtyping.factory.definition_builder.build_instance(AST::Builtin::Object.module_name)
+        case annotations.self_type
+        when AST::Types::Name::Instance
+          module_name = annotations.self_type.name
+          module_type = AST::Types::Name::Singleton.new(name: module_name)
+          instance_type = annotations.self_type
+        when AST::Types::Name::Singleton
+          module_name = annotations.self_type.name
+          module_type = annotations.self_type
+          instance_type = annotations.self_type
+        else
+          module_name = AST::Builtin::Object.module_name
+          module_type = AST::Builtin::Object.module_type
+          instance_type = AST::Builtin::Object.instance_type
+        end
+
+        definition = subtyping.factory.definition_builder.build_instance(module_name)
 
         const_env = TypeInference::ConstantEnv.new(
           factory: subtyping.factory,
@@ -321,17 +336,17 @@ module Steep
         context = TypeInference::Context.new(
           block_context: nil,
           module_context: TypeInference::Context::ModuleContext.new(
-            instance_type: AST::Builtin::Object.instance_type,
-            module_type: AST::Builtin::Object.module_type,
+            instance_type: instance_type,
+            module_type: module_type,
             implement_name: nil,
             nesting: nil,
-            class_name: AST::Builtin::Object.module_name,
-            instance_definition: subtyping.factory.definition_builder.build_instance(AST::Builtin::Object.module_name),
-            module_definition: subtyping.factory.definition_builder.build_singleton(AST::Builtin::Object.module_name)
+            class_name: module_name,
+            instance_definition: subtyping.factory.definition_builder.build_instance(module_name),
+            module_definition: subtyping.factory.definition_builder.build_singleton(module_name)
           ),
           method_context: nil,
           break_context: nil,
-          self_type: AST::Builtin::Object.instance_type,
+          self_type: instance_type,
           type_env: type_env,
           call_context: TypeInference::MethodCall::TopLevelContext.new,
           variable_context: TypeInference::Context::TypeVariableContext.empty

data/lib/steep/source.rb

@@ -498,6 +498,8 @@ module Steep
             # Skip
           when :kwargs
             # skip
+          when :when
+            # skip
           when :pair
             key, value = node.children
             key = insert_type_node(key, comments.except(last_line))
@@ -644,7 +646,7 @@ module Steep
         receiver_node, name, _, location = deconstruct_send_node!(send_node)
 
         if receiver_node
-          if location.dot
+          if location.dot && location.selector
             location.selector.line
           end
         else

data/lib/steep/subtyping/check.rb

@@ -285,11 +285,6 @@ module Steep
             )
           end
 
-        when relation.sub_type.is_a?(AST::Types::Self) && !self_type.is_a?(AST::Types::Self)
-          Expand(relation) do
-            check_type(Relation.new(sub_type: self_type, super_type: relation.super_type))
-          end
-
         when relation.sub_type.is_a?(AST::Types::Instance) && !instance_type.is_a?(AST::Types::Instance)
           Expand(relation) do
             check_type(Relation.new(sub_type: instance_type, super_type: relation.super_type))
@@ -419,6 +414,11 @@ module Steep
             end
           end
 
+        when relation.sub_type.is_a?(AST::Types::Self) && !self_type.is_a?(AST::Types::Self)
+          Expand(relation) do
+            check_type(Relation.new(sub_type: self_type, super_type: relation.super_type))
+          end
+
         when relation.super_type.is_a?(AST::Types::Name::Interface)
           Expand(relation) do
             check_interface(
@@ -493,7 +493,7 @@ module Steep
           end
 
         when relation.sub_type.is_a?(AST::Types::Tuple) && relation.super_type.is_a?(AST::Types::Tuple)
-          if relation.sub_type.types.size >= relation.super_type.types.size
+          if relation.sub_type.types.size == relation.super_type.types.size
             pairs = relation.sub_type.types.take(relation.super_type.types.size).zip(relation.super_type.types)
 
             All(relation) do |result|
@@ -541,21 +541,28 @@ module Steep
 
         when relation.sub_type.is_a?(AST::Types::Record) && relation.super_type.is_a?(AST::Types::Record)
           All(relation) do |result|
+            unchecked_keys = Set.new(relation.sub_type.elements.each_key)
+
             relation.super_type.elements.each_key do |key|
-              super_element_type = relation.super_type.elements[key]
+              super_element_type = relation.super_type.elements.fetch(key) #: AST::Types::t
+              sub_element_type = relation.sub_type.elements.fetch(key, nil) #: AST::Types::t?
 
-              if relation.sub_type.elements.key?(key)
-                sub_element_type = relation.sub_type.elements[key]
+              if relation.super_type.required?(key)
+                rel = Relation.new(sub_type: sub_element_type || AST::Builtin.nil_type, super_type: super_element_type)
+                result.add(rel) { check_type(rel) }
               else
-                if relation.super_type.required?(key)
-                  sub_element_type = AST::Builtin.nil_type
+                # If the key is optional, it's okay to not have the key
+                if sub_element_type
+                  rel = Relation.new(sub_type: sub_element_type, super_type: super_element_type)
+                  result.add(rel) { check_type(rel) }
                 end
               end
 
-              if sub_element_type
-                rel = Relation.new(sub_type: sub_element_type, super_type: super_element_type)
-                result.add(rel) { check_type(rel) }
-              end
+              unchecked_keys.delete(key)
+            end
+
+            unless unchecked_keys.empty?
+              return Failure(relation, Result::Failure::UnknownPairError.new(relation: relation))
             end
           end
 
@@ -601,7 +608,6 @@ module Steep
           Expand(relation) do
             check_type(Relation.new(sub_type: relation.sub_type.back_type, super_type: relation.super_type))
           end
-
         else
           Failure(relation, Result::Failure::UnknownPairError.new(relation: relation))
         end