steep 0.37.0 → 0.42.0

data/lib/steep/server/type_check_worker.rb

@@ -0,0 +1,122 @@
+module Steep
+  module Server
+    class TypeCheckWorker < BaseWorker
+      attr_reader :project, :assignment, :service
+      attr_reader :commandline_args
+
+      TypeCheckJob = Class.new
+      WorkspaceSymbolJob = Struct.new(:query, :id, keyword_init: true)
+      StatsJob = Struct.new(:id, keyword_init: true)
+
+      include ChangeBuffer
+
+      def initialize(project:, reader:, writer:, assignment:, commandline_args:)
+        super(project: project, reader: reader, writer: writer)
+
+        @assignment = assignment
+        @service = Services::TypeCheckService.new(project: project, assignment: assignment)
+        @buffered_changes = {}
+        @mutex = Mutex.new()
+        @queue = Queue.new
+        @commandline_args = commandline_args
+      end
+
+      def handle_request(request)
+        case request[:method]
+        when "initialize"
+          load_files(project: project, commandline_args: commandline_args)
+          queue << TypeCheckJob.new()
+          writer.write({ id: request[:id], result: nil})
+        when "textDocument/didChange"
+          collect_changes(request)
+          queue << TypeCheckJob.new()
+        when "workspace/symbol"
+          query = request[:params][:query]
+          queue << WorkspaceSymbolJob.new(id: request[:id], query: query)
+        when "workspace/executeCommand"
+          case request[:params][:command]
+          when "steep/stats"
+            queue << StatsJob.new(id: request[:id])
+          end
+        end
+      end
+
+      def handle_job(job)
+        case job
+        when TypeCheckJob
+          pop_buffer() do |changes|
+            break if changes.empty?
+
+            formatter = Diagnostic::LSPFormatter.new()
+
+            service.update(changes: changes) do |path, diagnostics|
+              if target = project.target_for_source_path(path)
+                diagnostics = diagnostics.select {|diagnostic| target.options.error_to_report?(diagnostic) }
+              end
+
+              writer.write(
+                method: :"textDocument/publishDiagnostics",
+                params: LSP::Interface::PublishDiagnosticsParams.new(
+                  uri: URI.parse(project.absolute_path(path).to_s).tap {|uri| uri.scheme = "file"},
+                  diagnostics: diagnostics.map {|diagnostic| formatter.format(diagnostic) }.uniq
+                )
+              )
+            end
+          end
+        when WorkspaceSymbolJob
+          writer.write(
+            id: job.id,
+            result: workspace_symbol_result(job.query)
+          )
+        when StatsJob
+          writer.write(
+            id: job.id,
+            result: stats_result().map(&:as_json)
+          )
+        end
+      end
+
+      def workspace_symbol_result(query)
+        Steep.measure "Generating workspace symbol list for query=`#{query}`" do
+          indexes = project.targets.map {|target| service.signature_services[target.name].latest_rbs_index }
+
+          provider = Index::SignatureSymbolProvider.new()
+          provider.indexes.push(*indexes)
+
+          symbols = provider.query_symbol(query, assignment: assignment)
+
+          symbols.map do |symbol|
+            LSP::Interface::SymbolInformation.new(
+              name: symbol.name,
+              kind: symbol.kind,
+              location: symbol.location.yield_self do |location|
+                path = Pathname(location.buffer.name)
+                {
+                  uri: URI.parse(project.absolute_path(path).to_s).tap {|uri| uri.scheme = "file" },
+                  range: {
+                    start: { line: location.start_line - 1, character: location.start_column },
+                    end: { line: location.end_line - 1, character: location.end_column }
+                  }
+                }
+              end,
+              container_name: symbol.container_name
+            )
+          end
+        end
+      end
+
+      def stats_result
+        calculator = Services::StatsCalculator.new(service: service)
+
+        project.targets.each.with_object([]) do |target, stats|
+          service.source_files.each_value do |file|
+            next unless assignment =~ file.path
+            next unless target.possible_source_file?(file.path)
+
+            stats << calculator.calc_stats(target, file: file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/steep/server/worker_process.rb

@@ -16,19 +16,21 @@ module Steep
         @name = name
       end
 
-      def self.spawn_worker(type, name:, steepfile:)
+      def self.spawn_worker(type, name:, steepfile:, options: [], delay_shutdown: false)
         log_level = %w(debug info warn error fatal unknown)[Steep.logger.level]
         command = case type
-                  when :code
-                    ["steep", "worker", "--code", "--name=#{name}", "--log-level=#{log_level}", "--steepfile=#{steepfile}"]
-                  when :signature
-                    ["steep", "worker", "--signature", "--name=#{name}", "--log-level=#{log_level}", "--steepfile=#{steepfile}"]
                   when :interaction
-                    ["steep", "worker", "--interaction", "--name=#{name}", "--log-level=#{log_level}", "--steepfile=#{steepfile}"]
+                    ["steep", "worker", "--interaction", "--name=#{name}", "--log-level=#{log_level}", "--steepfile=#{steepfile}", *options]
+                  when :typecheck
+                    ["steep", "worker", "--typecheck", "--name=#{name}", "--log-level=#{log_level}", "--steepfile=#{steepfile}", *options]
                   else
-                    raise
+                    raise "Unknown type: #{type}"
                   end
 
+        if delay_shutdown
+          command << "--delay-shutdown"
+        end
+
         stdin, stdout, thread = Open3.popen2(*command, pgroup: true)
         stderr = nil
 
@@ -38,9 +40,13 @@ module Steep
         new(reader: reader, writer: writer, stderr: stderr, wait_thread: thread, name: name)
       end
 
-      def self.spawn_code_workers(steepfile:, count: [Etc.nprocessors-3, 1].max)
+      def self.spawn_typecheck_workers(steepfile:, args:, count: [Etc.nprocessors - 1, 1].max, delay_shutdown: false)
         count.times.map do |i|
-          spawn_worker(:code, name: "code@#{i}", steepfile: steepfile)
+          spawn_worker(:typecheck,
+                       name: "typecheck@#{i}",
+                       steepfile: steepfile,
+                       options: ["--max-index=#{count}", "--index=#{i}", *args],
+                       delay_shutdown: delay_shutdown)
         end
       end
 
@@ -52,12 +58,8 @@ module Steep
         reader.read(&block)
       end
 
-      def shutdown
-        self << { method: :shutdown, params: nil }
-        self << { method: :exit, params: nil }
-
-        writer.io.close()
-        @wait_thread.join()
+      def kill
+        Process.kill(:TERM, @wait_thread.pid)
       end
     end
   end

data/lib/steep/{project → services}/completion_provider.rb

@@ -1,5 +1,5 @@
 module Steep
-  class Project
+  module Services
     class CompletionProvider
       Position = Struct.new(:line, :column, keyword_init: true) do
         def -(size)
@@ -51,13 +51,13 @@ module Steep
         @modified_text = text
 
         Steep.measure "parsing" do
-          @source = SourceFile
+          @source = Source
                       .parse(text, path: path, factory: subtyping.factory)
                       .without_unrelated_defs(line: line, column: column)
         end
 
         Steep.measure "typechecking" do
-          @typing = SourceFile.type_check(source, subtyping: subtyping)
+          @typing = TypeCheckService.type_check(source: source, subtyping: subtyping)
         end
       end
 

data/lib/steep/services/content_change.rb

@@ -0,0 +1,61 @@
+module Steep
+  module Services
+    class ContentChange
+      class Position
+        attr_reader :line, :column
+
+        def initialize(line:, column:)
+          @line = line
+          @column = column
+        end
+
+        def ==(other)
+          other.is_a?(Position) && other.line == line && other.column == column
+        end
+
+        alias eql? ==
+
+        def hash
+          self.class.hash ^ line ^ column
+        end
+      end
+
+      attr_reader :range, :text
+
+      def initialize(range: nil, text:)
+        @range = range
+        @text = text
+      end
+
+      def ==(other)
+        other.is_a?(ContentChange) && other.range == range && other.text == text
+      end
+
+      alias eql? ==
+
+      def hash
+        self.class.hash ^ range.hash ^ text.hash
+      end
+
+      def self.string(string)
+        new(text: string)
+      end
+
+      def apply_to(text)
+        if range
+          text = text.dup
+          start_pos, end_pos = range
+
+          buf = RBS::Buffer.new(name: :_, content: text)
+          start_pos = buf.loc_to_pos([start_pos.line, start_pos.column])
+          end_pos = buf.loc_to_pos([end_pos.line, end_pos.column])
+
+          text[start_pos...end_pos] = self.text
+          text
+        else
+          self.text
+        end
+      end
+    end
+  end
+end

data/lib/steep/services/file_loader.rb

@@ -0,0 +1,48 @@
+module Steep
+  module Services
+    class FileLoader
+      attr_reader :base_dir
+
+      def initialize(base_dir:)
+        @base_dir = base_dir
+      end
+
+      def each_path_in_patterns(pattern, commandline_patterns = [])
+        pats = commandline_patterns.empty? ? pattern.patterns : commandline_patterns
+
+        pats.each do |path|
+          absolute_path = base_dir + path
+
+          if absolute_path.file?
+            yield absolute_path.relative_path_from(base_dir)
+          else
+            files = if absolute_path.directory?
+                      Pathname.glob("#{absolute_path}/**/*#{pattern.ext}")
+                    else
+                      Pathname.glob(absolute_path)
+                    end
+
+            files.sort.each do |source_path|
+              if source_path.file?
+                relative_path = source_path.relative_path_from(base_dir)
+                unless pattern.ignore?(relative_path)
+                  yield relative_path
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def load_changes(pattern, command_line_patterns = [], changes:)
+        each_path_in_patterns(pattern, command_line_patterns) do |path|
+          unless changes.key?(path)
+            changes[path] = [ContentChange.string((base_dir + path).read)]
+          end
+        end
+
+        changes
+      end
+    end
+  end
+end

data/lib/steep/{project → services}/hover_content.rb

@@ -1,5 +1,5 @@
 module Steep
-  class Project
+  module Services
     class HoverContent
       TypeContent = Struct.new(:node, :type, :location, keyword_init: true)
       VariableContent = Struct.new(:node, :name, :type, :location, keyword_init: true)
@@ -15,10 +15,14 @@ module Steep
       InstanceMethodName = Struct.new(:class_name, :method_name)
       SingletonMethodName = Struct.new(:class_name, :method_name)
 
-      attr_reader :project
+      attr_reader :service
 
-      def initialize(project:)
-        @project = project
+      def initialize(service:)
+        @service = service
+      end
+
+      def project
+        service.project
       end
 
       def method_definition_for(factory, type_name, singleton_method: nil, instance_method: nil)
@@ -36,17 +40,10 @@ module Steep
         end
       end
 
-      def typecheck(target, path:, line:, column:)
-        target.type_check(target_sources: [], validate_signatures: false)
-
-        case (status = target.status)
-        when Project::Target::TypeCheckStatus
-          subtyping = status.subtyping
-          source = SourceFile
-                     .parse(target.source_files[path].content, path: path, factory: subtyping.factory)
-                     .without_unrelated_defs(line: line, column: column)
-          SourceFile.type_check(source, subtyping: subtyping)
-        end
+      def typecheck(target, path:, content:, line:, column:)
+        subtyping = service.signature_services[target.name].current_subtyping or return
+        source = Source.parse(content, path: path, factory: subtyping.factory)
+        Services::TypeCheckService.type_check(source: source, subtyping: subtyping)
       rescue
         nil
       end
@@ -55,7 +52,8 @@ module Steep
         target = project.target_for_source_path(path)
 
         if target
-          typing = typecheck(target, path: path, line: line, column: column) or return
+          file = service.source_files[path]
+          typing = typecheck(target, path: path, content: file.content, line: line, column: column) or return
 
           node, *parents = typing.source.find_nodes(line: line, column: column)
 

data/lib/steep/services/path_assignment.rb

@@ -0,0 +1,29 @@
+module Steep
+  module Services
+    class PathAssignment
+      attr_reader :index, :max_index, :cache
+
+      def initialize(index:, max_index:)
+        @index = index
+        @max_index = max_index
+        @cache = {}
+      end
+
+      def self.all
+        new(index: 0, max_index: 1)
+      end
+
+      def =~(path)
+        path = path.to_s
+
+        if cache.key?(path)
+          cache[path]
+        else
+          value = Digest::MD5.hexdigest(path).hex % max_index == index
+          cache[path] = value
+          value
+        end
+      end
+    end
+  end
+end

data/lib/steep/services/signature_service.rb

@@ -0,0 +1,369 @@
+module Steep
+  module Services
+    class SignatureService
+      attr_reader :status
+
+      class SyntaxErrorStatus
+        attr_reader :files, :changed_paths, :diagnostics, :last_builder
+
+        def initialize(files:, changed_paths:, diagnostics:, last_builder:)
+          @files = files
+          @changed_paths = changed_paths
+          @diagnostics = diagnostics
+          @last_builder = last_builder
+        end
+
+        def rbs_index
+          @rbs_index ||= Index::RBSIndex.new().tap do |index|
+            builder = Index::RBSIndex::Builder.new(index: index)
+            builder.env(last_builder.env)
+          end
+        end
+      end
+
+      class AncestorErrorStatus
+        attr_reader :files, :changed_paths, :diagnostics, :last_builder
+
+        def initialize(files:, changed_paths:, diagnostics:, last_builder:)
+          @files = files
+          @changed_paths = changed_paths
+          @diagnostics = diagnostics
+          @last_builder = last_builder
+        end
+
+        def rbs_index
+          @rbs_index ||= Index::RBSIndex.new().tap do |index|
+            builder = Index::RBSIndex::Builder.new(index: index)
+            builder.env(last_builder.env)
+          end
+        end
+      end
+
+      class LoadedStatus
+        attr_reader :files, :builder
+
+        def initialize(files:, builder:)
+          @files = files
+          @builder = builder
+        end
+
+        def subtyping
+          @subtyping ||= Subtyping::Check.new(factory: AST::Types::Factory.new(builder: builder))
+        end
+
+        def rbs_index
+          @rbs_index ||= Index::RBSIndex.new().tap do |index|
+            builder = Index::RBSIndex::Builder.new(index: index)
+            builder.env(self.builder.env)
+          end
+        end
+      end
+
+      FileStatus = Struct.new(:path, :content, :decls, keyword_init: true)
+
+      def initialize(env:)
+        builder = RBS::DefinitionBuilder.new(env: env)
+        @status = LoadedStatus.new(builder: builder, files: {})
+      end
+
+      def self.load_from(loader)
+        env = RBS::Environment.from_loader(loader).resolve_type_names
+        new(env: env)
+      end
+
+      def each_rbs_path(&block)
+        if block
+          latest_env.buffers.each do |buffer|
+            unless files.key?(buffer.name)
+              yield Pathname(buffer.name)
+            end
+          end
+
+          files.each_key(&block)
+        else
+          enum_for :each_rbs_path
+        end
+      end
+
+      def files
+        status.files
+      end
+
+      def pending_changed_paths
+        case status
+        when LoadedStatus
+          Set[]
+        when SyntaxErrorStatus, AncestorErrorStatus
+          Set.new(status.changed_paths)
+        end
+      end
+
+      def latest_env
+        latest_builder.env
+      end
+
+      def latest_builder
+        case status
+        when LoadedStatus
+          status.builder
+        when SyntaxErrorStatus, AncestorErrorStatus
+          status.last_builder
+        end
+      end
+
+      def latest_rbs_index
+        status.rbs_index
+      end
+
+      def current_subtyping
+        if status.is_a?(LoadedStatus)
+          status.subtyping
+        end
+      end
+
+      def apply_changes(files, changes)
+        Steep.logger.tagged "#apply_changes" do
+          Steep.measure2 "Applying change" do |sampler|
+            changes.each.with_object({}) do |(path, cs), update|
+              sampler.sample "#{path}" do
+                old_text = files[path]&.content
+                content = cs.inject(old_text || "") {|text, change| change.apply_to(text) }
+
+                buffer = RBS::Buffer.new(name: path, content: content)
+
+                update[path] = begin
+                                 FileStatus.new(path: path, content: content, decls: RBS::Parser.parse_signature(buffer))
+                               rescue RBS::ParsingError => exn
+                                 FileStatus.new(path: path, content: content, decls: exn)
+                               end
+              end
+            end
+          end
+        end
+      end
+
+      def update(changes)
+        Steep.logger.tagged "#update" do
+          updates = apply_changes(files, changes)
+          paths = Set.new(updates.each_key)
+          paths.merge(pending_changed_paths)
+
+          if updates.each_value.any? {|file| file.decls.is_a?(RBS::ParsingError) }
+            diagnostics = []
+
+            updates.each_value do |file|
+              if file.decls.is_a?(RBS::ParsingError)
+                # factory is not used here because the error is a syntax error.
+                diagnostics << Diagnostic::Signature.from_rbs_error(file.decls, factory: nil)
+              end
+            end
+
+            @status = SyntaxErrorStatus.new(
+              files: self.files.merge(updates),
+              diagnostics: diagnostics,
+              last_builder: latest_builder,
+              changed_paths: paths
+            )
+          else
+            files = self.files.merge(updates)
+            updated_files = paths.each.with_object({}) do |path, hash|
+              hash[path] = files[path]
+            end
+            result =
+              Steep.measure "#update_env with updated #{paths.size} files" do
+                update_env(updated_files, paths: paths)
+              end
+
+            @status = case result
+                      when Array
+                        AncestorErrorStatus.new(
+                          changed_paths: paths,
+                          last_builder: latest_builder,
+                          diagnostics: result,
+                          files: files
+                        )
+                      when RBS::DefinitionBuilder::AncestorBuilder
+                        builder2 = update_builder(ancestor_builder: result, paths: paths)
+                        LoadedStatus.new(builder: builder2, files: files)
+                      end
+          end
+        end
+      end
+
+      def update_env(updated_files, paths:)
+        Steep.logger.tagged "#update_env" do
+          errors = []
+          new_decls = Set[].compare_by_identity
+
+          env =
+            Steep.measure "Deleting out of date decls" do
+              latest_env.reject do |decl|
+                if decl.location
+                  paths.include?(decl.location.buffer.name)
+                end
+              end
+            end
+
+          Steep.measure "Loading new decls" do
+            updated_files.each_value do |content|
+              if content.decls.is_a?(RBS::ErrorBase)
+                errors << content.decls
+              else
+                begin
+                  content.decls.each do |decl|
+                    env << decl
+                    new_decls << decl
+                  end
+                rescue RBS::LoadingError => exn
+                  errors << exn
+                end
+              end
+            end
+          end
+
+          Steep.measure "validate type params" do
+            begin
+              env.validate_type_params
+            rescue RBS::LoadingError => exn
+              errors << exn
+            end
+          end
+
+          unless errors.empty?
+            return errors.map {|error|
+              # Factory will not be used because of the possible error types.
+              Diagnostic::Signature.from_rbs_error(error, factory: nil)
+            }
+          end
+
+          Steep.measure "resolve type names with #{new_decls.size} top-level decls" do
+            env = env.resolve_type_names(only: new_decls)
+          end
+
+          builder = RBS::DefinitionBuilder::AncestorBuilder.new(env: env)
+
+          Steep.measure("Pre-loading one ancestors") do
+            builder.env.class_decls.each_key do |type_name|
+              rescue_rbs_error(errors) { builder.one_instance_ancestors(type_name) }
+              rescue_rbs_error(errors) { builder.one_singleton_ancestors(type_name) }
+            end
+            builder.env.interface_decls.each_key do |type_name|
+              rescue_rbs_error(errors) { builder.one_interface_ancestors(type_name) }
+            end
+          end
+
+          unless errors.empty?
+            # Builder won't be used.
+            factory = AST::Types::Factory.new(builder: nil)
+            return errors.map {|error| Diagnostic::Signature.from_rbs_error(error, factory: factory) }
+          end
+
+          builder
+        end
+      end
+
+      def rescue_rbs_error(errors)
+        begin
+          yield
+        rescue RBS::ErrorBase => exn
+          errors << exn
+        end
+      end
+
+      def update_builder(ancestor_builder:, paths:)
+        Steep.measure "#update_builder with #{paths.size} files" do
+          changed_names = Set[]
+
+          old_definition_builder = latest_builder
+          old_env = old_definition_builder.env
+          old_names = type_names(paths: paths, env: old_env)
+          old_ancestor_builder = old_definition_builder.ancestor_builder
+          old_graph = RBS::AncestorGraph.new(env: old_env, ancestor_builder: old_ancestor_builder)
+          add_descendants(graph: old_graph, names: old_names, set: changed_names)
+          add_nested_decls(env: old_env, names: old_names, set: changed_names)
+
+          new_env = ancestor_builder.env
+          new_ancestor_builder = ancestor_builder
+          new_names = type_names(paths: paths, env: new_env)
+          new_graph = RBS::AncestorGraph.new(env: new_env, ancestor_builder: new_ancestor_builder)
+          add_descendants(graph: new_graph, names: new_names, set: changed_names)
+          add_nested_decls(env: new_env, names: new_names, set: changed_names)
+
+          old_definition_builder.update(
+            env: new_env,
+            ancestor_builder: new_ancestor_builder,
+            except: changed_names
+          )
+        end
+      end
+
+      def type_names(paths:, env:)
+        env.declarations.each.with_object(Set[]) do |decl, set|
+          if decl.location
+            if paths.include?(decl.location.buffer.name)
+              type_name_from_decl(decl, set: set)
+            end
+          end
+        end
+      end
+
+      def type_name_from_decl(decl, set:)
+        case decl
+        when RBS::AST::Declarations::Class, RBS::AST::Declarations::Module, RBS::AST::Declarations::Interface
+          set << decl.name
+
+          decl.members.each do |member|
+            if member.is_a?(RBS::AST::Declarations::Base)
+              type_name_from_decl(member, set: set)
+            end
+          end
+        when RBS::AST::Declarations::Alias
+          set << decl.name
+        end
+      end
+
+      def add_descendants(graph:, names:, set:)
+        set.merge(names)
+        names.each do |name|
+          case
+          when name.interface?
+            graph.each_descendant(RBS::AncestorGraph::InstanceNode.new(type_name: name)) do |node|
+              set << node.type_name
+            end
+          when name.class?
+            graph.each_descendant(RBS::AncestorGraph::InstanceNode.new(type_name: name)) do |node|
+              set << node.type_name
+            end
+            graph.each_descendant(RBS::AncestorGraph::SingletonNode.new(type_name: name)) do |node|
+              set << node.type_name
+            end
+          end
+        end
+      end
+
+      def add_nested_decls(env:, names:, set:)
+        tops = names.each.with_object(Set[]) do |name, tops|
+          unless name.namespace.empty?
+            tops << name.namespace.path[0]
+          end
+        end
+
+        env.class_decls.each_key do |name|
+          unless name.namespace.empty?
+            if tops.include?(name.namespace.path[0])
+              set << name
+            end
+          end
+        end
+
+        env.interface_decls.each_key do |name|
+          unless name.namespace.empty?
+            if tops.include?(name.namespace.path[0])
+              set << name
+            end
+          end
+        end
+      end
+    end
+  end
+end